sunholo 0.79.0__py3-none-any.whl → 0.79.2__py3-none-any.whl

sunholo/llamaindex/import_files.py

@@ -3,11 +3,13 @@ try:
 except ImportError:
     rag = None
 
-from ..logging import log
+from ..custom_logging import log
 from ..utils import ConfigManager
 from ..vertex import init_vertex
 from .get_files import fetch_corpus
+from .llamaindex_class import LlamaIndexVertexCorpusManager
 from ..components import load_memories
+
     
 
 def do_llamaindex(message_data, metadata, vector_name):
@@ -50,7 +52,6 @@ def do_llamaindex(message_data, metadata, vector_name):
 
     global_project_id = gcp_config.get('project_id')
     global_location = gcp_config.get('location')
-    global_rag_id = gcp_config.get('rag_id')
     #global_data_store_id = gcp_config.get('data_store_id')
 
     memories = load_memories(vector_name)
@@ -70,26 +71,32 @@ def do_llamaindex(message_data, metadata, vector_name):
                 rag_id = value.get('rag_id')
                 project_id = gcp_config.get('project_id')
                 location = gcp_config.get('location')
-                try:
-                    corpus = fetch_corpus(
-                        project_id=project_id or global_project_id,
-                        location=location or global_location,
-                        rag_id=rag_id or global_rag_id
-                    )
-                except Exception as err:
-                    log.warning(f"Failed to fetch LlamaIndex corpus: {err=}")
-                    continue
+
+                if rag_id:    
+                    try:
+                        corpus = fetch_corpus(
+                            project_id=project_id or global_project_id,
+                            location=location or global_location,
+                            rag_id=rag_id
+                        )
+                    except Exception as err:
+                        log.warning(f"Failed to fetch LlamaIndex corpus from rag_id: {err=}")
+                        continue
+                else:
+                    try:
+                        log.info("Using vertex llamaindex with own rag_id created via VAC name")
+                        manager = LlamaIndexVertexCorpusManager(config, project_id=project_id, location=location)
+                        # create or get existing:
+                        corpus = manager.create_corpus(vector_name)
+                    except Exception as err:
+                        log.warning(f"Failed to fetch LlamaIndex corpus from display_name: {err=}")
+                        continue
                 
                 corpuses.append(corpus)
+                
     if not corpuses:
         log.info("No Vertex Llamaindex RAG corpus to import data")
         return None
-    
-    try:
-        corpura = rag.list_corpora()
-        log.info(f"All Corpora: {corpura} - {type(corpura)}")
-    except Exception as err:
-        log.warning(f"Could not list any corpora - {str(err)}")
 
     log.info(f"Found llamaindex corpus: {corpuses}")
 

sunholo/llamaindex/llamaindex_class.py

@@ -0,0 +1,274 @@
+import os
+import tempfile
+
+try:
+    from vertexai.preview import rag
+except ImportError:
+    rag = None
+
+from ..utils import ConfigManager
+from ..custom_logging import log
+
+try:
+    from ..cli.sun_rich import console
+except ImportError:
+    console = None
+
+class LlamaIndexVertexCorpusManager:
+    """
+    A manager class for handling RAG corpora operations with Vertex AI.
+
+    Attributes:
+        config (ConfigManager): Configuration manager for fetching project settings.
+        project_id (str): GCP project ID.
+        location (str): GCP location.
+    """
+
+    def __init__(self, config: ConfigManager = None, project_id: str = None, location: str = None):
+        """
+        Initializes the LlamaIndexVertexCorpusManager.
+
+        Args:
+            config (ConfigManager): Configuration manager for fetching project settings.
+            project_id (str): GCP project ID.
+            location (str): GCP location.
+
+        Raises:
+            ImportError: If vertexai is not installed.
+        """
+        from ..vertex.init import init_vertex
+        if rag is None:
+            raise ImportError("You must install vertexai via `pip install sunholo[gcp]`")
+        
+        self.config = config
+        self.project_id = project_id
+        self.location = location
+
+        if config:
+            self.project_id = self.config.vacConfig('project_id') or project_id
+            self.location = self.config.vacConfig('location') or location
+
+        init_vertex(location=self.location, project_id=self.project_id)
+
+    def upload_text(self, text: str, corpus_display_name: str, description: str = None):
+        """
+        Uploads a text string to a specified corpus by saving it to a temporary file first.
+
+        Args:
+            text (str): The text content to upload.
+            corpus_display_name (str): The display name of the corpus.
+            description (str, optional): Description of the text upload.
+
+        Returns:
+            The uploaded file object.
+        """
+        # Create a temporary file and write the text to it
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.txt') as temp_file:
+            temp_file.write(text.encode('utf-8'))
+            temp_filename = temp_file.name
+        
+        try:
+            # Upload the temporary file
+            uploaded_file = self.upload_file(temp_filename, corpus_display_name, description or text[:50])
+        finally:
+            # Clean up the temporary file
+            os.remove(temp_filename)
+
+        return uploaded_file
+    
+    def upload_file(self, filename:str, corpus_display_name:str, description:str=None):
+        corpus = self.find_corpus_from_list(corpus_display_name)
+
+        rag_file = rag.upload_file(
+            corpus_name=corpus.name,
+            path=filename,
+            display_name=filename,
+            description=description or f"Upload for {filename}",
+        )
+        log.info("Uploaded file: {rag_file}")
+
+        return rag_file
+    
+    def import_files(self, file_imports:list[str], corpus_display_name:str):
+        corpus = self.find_corpus_from_list(corpus_display_name)
+
+        log.info(f"Importing files: {file_imports} into {corpus.name}")
+        response = rag.import_files(
+            corpus_name=corpus.name,
+            paths=file_imports,
+            chunk_size=512,  # Optional
+            chunk_overlap=100,  # Optional
+            max_embedding_requests_per_min=900,  # Optional
+        )
+        log.info(f"Imported files: {response}")
+
+        return response
+    
+    def list_corpora(self):
+        """
+        List all VertexAI Corpus for the project/location
+        """
+        return rag.list_corpora()
+    
+    def find_corpus_from_list(self, display_name: str):
+        """
+        Finds a corpus from the list of corpora by its display name.
+
+        Args:
+            display_name (str): The display name of the corpus.
+
+        Returns:
+            The found corpus object if it exists, otherwise None.
+        """
+        corpora = self.list_corpora()
+        for corp in corpora:
+            if display_name == corp.display_name:
+                log.info(f"Found existing corpus with display name: {display_name}")
+                return corp
+        return None
+
+    def create_corpus(self, display_name: str, description: str = None):
+        """
+        Creates a new corpus or returns an existing one with the specified display name.
+
+        Args:
+            display_name (str): The display name of the corpus.
+            description (str, optional): Description of the corpus.
+
+        Returns:
+            The created or found corpus object.
+        """
+        corp = self.find_corpus_from_list(display_name)
+        if corp:
+            return corp
+        
+        corpus = rag.create_corpus(display_name=display_name, description=description or f"Corpus for {display_name}")
+        log.info(f"created Llamaindex corpus {corpus}")
+
+        return corpus
+
+    def delete_corpus(self, display_name: str):
+        """
+        Deletes a corpus by its display name.
+
+        Args:
+            display_name (str): The display name of the corpus.
+
+        Returns:
+            bool: True if the corpus was deleted, False otherwise.
+        """
+        corp = self.find_corpus_from_list(display_name)
+        if corp:
+            rag_id = corp.name
+            rag.delete_corpus(name=rag_id)
+            log.info(f"Deleted {rag_id}")
+            return True
+        
+        log.warning(f"Could not find a corp to delete with name {display_name}")
+        return False
+
+    def fetch_corpus(self, display_name: str):
+        """
+        Fetches a corpus by its display name.
+
+        Args:
+            display_name (str): The display name of the corpus.
+
+        Returns:
+            The fetched corpus object.
+
+        Raises:
+            ValueError: If the corpus with the specified display name does not exist.
+        """
+        corp = self.find_corpus_from_list(display_name)
+        if not corp:
+            raise ValueError(f"Could not find any corpus with display_name: {display_name}")
+        return rag.get_corpus(name=corp.name)
+
+def llamaindex_command(args):
+    if console is None:
+        raise ImportError("Need cli tools to use `sunholo llamaindex` - install via `pip install sunholo[cli]`")
+    
+    config = ConfigManager(args.vac)
+    manager = LlamaIndexVertexCorpusManager(config=config)
+    
+    if args.action == "create":
+        manager.create_corpus(display_name=args.display_name, description=args.description)
+    elif args.action == "delete":
+        manager.delete_corpus(display_name=args.display_name)
+    elif args.action == "fetch":
+        corpus = manager.fetch_corpus(display_name=args.display_name)
+        console.print(corpus)
+    elif args.action == "find":
+        corpus = manager.find_corpus_from_list(display_name=args.display_name)
+        console.print(corpus)
+    elif args.action == "list":
+        corpora = manager.list_corpora()
+        console.print(corpora)
+    elif args.action == "import_files":
+        manager.import_files(file_imports=args.file_imports, corpus_display_name=args.display_name)
+    elif args.action == "upload_file":
+        manager.upload_file(filename=args.filename, corpus_display_name=args.display_name, description=args.description)
+    elif args.action == "upload_text":
+        manager.upload_text(text=args.text, corpus_display_name=args.display_name, description=args.description)
+    else:
+        console.print(f"Unknown action: {args.action}")
+
+def setup_llamaindex_subparser(subparsers):
+    """
+    Sets up an argparse subparser for the 'llamaindex' command.
+
+    Args:
+        subparsers: The subparsers object from argparse.ArgumentParser().
+    """
+    # LlamaIndex subparser setup
+    llamaindex_parser = subparsers.add_parser('llamaindex', help='Manage LlamaIndex corpora')
+    llamaindex_subparsers = llamaindex_parser.add_subparsers(dest='action', help='LlamaIndex subcommands')
+
+    # LlamaIndex create command
+    create_parser = llamaindex_subparsers.add_parser('create', help='Create a new corpus')
+    create_parser.add_argument('display_name', help='The name of the corpus')
+    create_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+    create_parser.add_argument('--description', help='Description of the corpus', default=None)
+
+    # LlamaIndex delete command
+    delete_parser = llamaindex_subparsers.add_parser('delete', help='Delete a corpus')
+    delete_parser.add_argument('display_name', help='The name of the corpus')
+    delete_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+
+    # LlamaIndex fetch command
+    fetch_parser = llamaindex_subparsers.add_parser('fetch', help='Fetch a corpus')
+    fetch_parser.add_argument('display_name', help='The name of the corpus')
+    fetch_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+
+    # LlamaIndex find command
+    find_parser = llamaindex_subparsers.add_parser('find', help='Find a corpus')
+    find_parser.add_argument('display_name', help='The name of the corpus')
+    find_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+
+    # LlamaIndex list command
+    list_parser = llamaindex_subparsers.add_parser('list', help='List all corpus')
+    list_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+
+    # LlamaIndex import_files command
+    import_files_parser = llamaindex_subparsers.add_parser('import_files', help='Import files from URLs to a corpus')
+    import_files_parser.add_argument('display_name', help='The name of the corpus')
+    import_files_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+    import_files_parser.add_argument('file_imports', nargs='+', help='The list of file URLs to import')
+
+    # LlamaIndex upload_file command
+    upload_file_parser = llamaindex_subparsers.add_parser('upload_file', help='Upload a local file to a corpus')
+    upload_file_parser.add_argument('display_name', help='The name of the corpus')
+    upload_file_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+    upload_file_parser.add_argument('filename', help='The local file path to upload')
+    upload_file_parser.add_argument('--description', help='Description of the file upload', default=None)
+
+    # LlamaIndex upload_text command
+    upload_text_parser = llamaindex_subparsers.add_parser('upload_text', help='Upload text to a corpus')
+    upload_text_parser.add_argument('display_name', help='The name of the corpus')
+    upload_text_parser.add_argument('vac', nargs='?', default="global", help='The VAC config to set it up for')
+    upload_text_parser.add_argument('text', help='The text content to upload')
+    upload_text_parser.add_argument('--description', help='Description of the text upload', default=None)
+
+    llamaindex_parser.set_defaults(func=llamaindex_command)
+    

sunholo/pubsub/process_pubsub.py

@@ -12,7 +12,7 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 import base64
-from ..logging import log
+from ..custom_logging import log
 from ..gcs.metadata import get_object_metadata
 
 

sunholo/pubsub/pubsub_manager.py

@@ -19,7 +19,7 @@ except ImportError:
     pubsub_v1 = None
 
 import json
-from ..logging import log
+from ..custom_logging import log
 from ..utils.gcp_project import get_gcp_project
 
 

sunholo/qna/parsers.py

@@ -12,7 +12,7 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-from sunholo.logging import log
+from sunholo.custom_logging import log
 
 def document_to_dict(document):
     return {

sunholo/qna/retry.py

@@ -13,7 +13,7 @@
 #   limitations under the License.
 import time
 import traceback
-from ..logging import log
+from ..custom_logging import log
 
 
 def retry_qna(qa_function, question, max_retries=1, initial_delay=5):

sunholo/streaming/content_buffer.py

@@ -17,7 +17,7 @@ from langchain.schema import LLMResult
 
 import threading
 import re
-from ..logging import log
+from ..custom_logging import log
 
 class ContentBuffer:
     """

sunholo/streaming/langserve.py

@@ -1,5 +1,5 @@
 import json
-from ..logging import log
+from ..custom_logging import log
 
 async def parse_langserve_token_async(token):
     """

sunholo/streaming/stream_lookup.py

@@ -1,5 +1,5 @@
 from ..utils import load_config_key
-from ..logging import log
+from ..custom_logging import log
 
 def can_agent_stream(agent_name: str):
 

sunholo/streaming/streaming.py

@@ -21,7 +21,7 @@ from .content_buffer import ContentBuffer, BufferStreamingStdOutCallbackHandler
 
 from ..qna.parsers import parse_output
 
-from ..logging import log
+from ..custom_logging import log
 from ..utils import load_config_key
 from ..utils.parsers import check_kwargs_support
 

sunholo/summarise/summarise.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 
 
 

sunholo/tools/web_browser.py

@@ -6,7 +6,7 @@ import urllib.parse
 import time
 from io import BytesIO
 
-from ..logging import log
+from ..custom_logging import log
 
 from ..utils.parsers import get_clean_website_name
 

sunholo/utils/config.py

@@ -36,7 +36,7 @@ def get_module_filepath(filepath: str):
     print(f'Absolute path: {abs_path}')
     ```
     """
-    from ..logging import log
+    from ..custom_logging import log
     
     # Get the root directory of this Python script
     dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
@@ -55,7 +55,7 @@ def load_all_configs():
     Files are expected to be either YAML or JSON and must contain a 'kind' key at the root.
     Caching is used to avoid reloading files within a 5-minute window.
     """
-    from ..logging import log
+    from ..custom_logging import log
 
     if not os.getenv("_CONFIG_FOLDER"):
         log.debug("_CONFIG_FOLDER is not set, using os.getcwd() instead")
@@ -102,7 +102,7 @@ def reload_config_file(config_file, filename):
     """
     Helper function to load a config file and update the cache.
     """
-    from ..logging import log
+    from ..custom_logging import log
     with open(config_file, 'r') as file:
         if filename.endswith('.json'):
             config = json.load(file)
@@ -134,7 +134,7 @@ def load_config(filename: str=None) -> tuple[dict, str]:
     ```
     """
     global config_cache
-    from ..logging import log
+    from ..custom_logging import log
     
     if filename is None:
         filename = os.getenv("_CONFIG_FILE", None)
@@ -194,7 +194,7 @@ def load_config_key(key: str, vector_name: str, kind: str):
     ```
     """
     # can't use sunholo.logging due to circular import
-    from ..logging import log
+    from ..custom_logging import log
 
     if kind != 'agentConfig':
         assert isinstance(key, str), f"key must be a string got a {type(key)}"

sunholo/utils/config_class.py

@@ -38,7 +38,7 @@ class ConfigManager:
         self.configs_by_kind = self.load_all_configs()
 
         test_agent = self.vacConfig("agent")
-        if not test_agent:
+        if not test_agent and self.vector_name != "global":
             print(f"WARNING: No vacConfig.agent found for {self.vector_name} - are you in right folder? {local_config_folder=} {self.config_folder=}")
 
     def load_all_configs(self):
@@ -49,7 +49,7 @@ class ConfigManager:
         Returns:
             dict: A dictionary of configurations grouped by their 'kind' key.
         """
-        from ..logging import log
+        from ..custom_logging import log
 
         log.debug(f"Loading all configs from folder: {self.config_folder} and local folder: {self.local_config_folder}")
         global_configs_by_kind = self._load_configs_from_folder(self.config_folder)
@@ -75,7 +75,7 @@ class ConfigManager:
         Returns:
             dict: A dictionary of configurations grouped by their 'kind' key.
         """
-        from ..logging import log
+        from ..custom_logging import log
 
         configs_by_kind = defaultdict(dict)
         current_time = datetime.now()
@@ -116,7 +116,7 @@ class ConfigManager:
         Returns:
             dict: The loaded configuration.
         """
-        from ..logging import log
+        from ..custom_logging import log
         with open(config_file, 'r') as file:
             if filename.endswith('.json'):
                 config = json.load(file)

sunholo/vertex/extensions_call.py

@@ -1,6 +1,6 @@
 from .extensions_class import VertexAIExtensions
 from ..utils import ConfigManager
-from ..logging import log
+from ..custom_logging import log
 import collections.abc
 import json
 

sunholo/vertex/extensions_class.py

@@ -4,7 +4,7 @@ except ImportError:
     extensions = None
 
 from .init import init_vertex
-from ..logging import log
+from ..custom_logging import log
 from ..utils.gcp_project import get_gcp_project
 from ..utils.gcp import is_running_on_cloudrun
 from ..auth import get_local_gcloud_token, get_cloud_run_token

sunholo/vertex/genai_functions.py

@@ -5,7 +5,7 @@ except ImportError:
 
 from .init import init_genai
 from .safety import genai_safety
-from ..logging import log
+from ..custom_logging import log
 import json
 from .type_dict_to_json import describe_typed_dict, openapi_to_typed_dict, is_typed_dict
 

sunholo/vertex/init.py

@@ -1,4 +1,4 @@
-from ..logging import log
+from ..custom_logging import log
 from ..utils.gcp_project import get_gcp_project
 from ..auth.refresh import get_default_email
 import os

sunholo/vertex/memory_tools.py

@@ -4,14 +4,15 @@ try:
 except ImportError:
     rag = None
 
-from ..logging import log
-from ..utils.config import load_config_key
+from ..custom_logging import log
 from ..components import load_memories
 from ..llamaindex.get_files import fetch_corpus
+from ..llamaindex.llamaindex_class import LlamaIndexVertexCorpusManager
 from ..discovery_engine.discovery_engine_client import DiscoveryEngineClient
 from ..utils.gcp_project import get_gcp_project
+from ..utils import ConfigManager
 
-def get_vertex_memories(vector_name):
+def get_vertex_memories(vector_name:str=None, config:ConfigManager=None):
     """
     Retrieves a LlamaIndex corpus from Vertex AI based on the provided Google Cloud configuration.
     
@@ -39,7 +40,12 @@ def get_vertex_memories(vector_name):
         print("Error fetching corpus:", str(e))
     ```
     """
-    gcp_config = load_config_key("gcp_config", vector_name=vector_name, kind="vacConfig")
+    if config is None:
+        if vector_name is None:
+            raise ValueError("config or vector_name are required")
+        config = ConfigManager(vector_name)
+
+    gcp_config = config.vacConfig("gcp_config")
 
     if not rag:
         raise ValueError("Need to install vertexai module via `pip install sunholo[gcp]`")
@@ -59,18 +65,22 @@ def get_vertex_memories(vector_name):
             if vectorstore == "llamaindex":
                 log.info(f"Found vectorstore {vectorstore}")
                 rag_id = value.get('rag_id')
-                if rag_id is None:
-                    raise ValueError("Must specify rag_id if using vectorstore: llamaindex")
-                
                 project_id = value.get('project_id') or gcp_config.get('project_id')
                 location = value.get('location') or gcp_config.get('location')
 
-                try:
+                if rag_id:
+                    log.info("Using rag_id for using vectorstore: llamaindex")
                     corpus = fetch_corpus(
-                        project_id=project_id or get_gcp_project(),
-                        location=location or global_location,
-                        rag_id=rag_id
-                    )
+                            project_id=project_id or get_gcp_project(),
+                            location=location or global_location,
+                            rag_id=rag_id
+                        )
+                else:
+                    log.info(f"Using display_name {vector_name} to derive rag_id")
+                    manager = LlamaIndexVertexCorpusManager(project_id=project_id, location=location)
+                    corpus = manager.find_corpus_from_list(vector_name)
+
+                try:
                     corpus_tool = Tool.from_retrieval(
                         retrieval=rag.Retrieval(
                             source=rag.VertexRagStore(
@@ -116,9 +126,14 @@ def get_vertex_memories(vector_name):
     
     return tools
 
-def get_google_search_grounding(vector_name):
+def get_google_search_grounding(vector_name:str=None, config:ConfigManager=None):
+    if config is None:
+        if vector_name is None:
+            raise ValueError("Must specify one of vector_name or config")
+        config = ConfigManager(vector_name)
+
     # can't have this and llamaindex memories?
-    ground = load_config_key("grounding", vector_name=vector_name, kind="vacConfig")
+    ground = config.vacConfig("grounding")
     if ground and ground.get("google_search"):
         gs_tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval())
         log.info(f"Got Search Tool: {gs_tool}")

{sunholo-0.79.0.dist-info → sunholo-0.79.2.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.79.0
+Version: 0.79.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.0.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.2.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0