sunholo 0.57.2__py3-none-any.whl → 0.58.2__py3-none-any.whl

sunholo/__init__.py

@@ -16,6 +16,7 @@ from . import pubsub
 from . import qna
 from . import streaming
 from . import utils
+from . import vertex
 import logging
 
 __all__ = ['agents', 

sunholo/agents/flask/qna_routes.py

@@ -61,7 +61,6 @@ def register_qna_routes(app, stream_interpreter, vac_interpreter):
                 model=vac_config.get("model") or vac_config.get("llm")
             )
 
-        @observe()
         def generate_response_content():
 
             for chunk in start_streaming_chat(question=all_input["user_input"],

sunholo/chunker/data_to_embed_pubsub.py

@@ -63,14 +63,14 @@ def data_to_embed_pubsub(data: dict):
     if metadata:
         metadata["vector_name"] = vector_name
 
-    if metadata.get("return_chunks"):
-        log.info("attributes.return_chunks=True detected, skipping process chunks queue")
-        output_list = []
-        if chunks:
-            for chunk in chunks:
-                output_list.append({"page_content": chunk.page_content, "metadata": chunk.metadata})
-                
-        return output_list
+        if metadata.get("return_chunks"):
+            log.info("attributes.return_chunks=True detected, skipping process chunks queue")
+            output_list = []
+            if chunks:
+                for chunk in chunks:
+                    output_list.append({"page_content": chunk.page_content, "metadata": chunk.metadata})
+                    
+            return output_list
 
     process_docs_chunks_vector_name(chunks, vector_name, metadata)
 

sunholo/cli/cli.py

@@ -1,59 +1,8 @@
 import argparse
-try:
-    from google.cloud.devtools import cloudbuild_v1
-except ImportError:
-    cloudbuild_v1 = None
 
-from ..logging import log
-
-def trigger_build(args):
-    """
-    Triggers a Google Cloud Build using an existing build trigger configured in GCP.
-
-    Args:
-        args: argparse.Namespace containing the command line arguments specified for the 'deploy' command.
-
-    Example:
-        trigger_build(args) where args contains project_id, trigger_id, repo_name, and branch_name.
-    """
-    if not cloudbuild_v1:
-        log.warning("Can't deploy - google-cloud-build not installed, enable via `pip install sunholo[gcp]")
-
-        return None
-
-    client = cloudbuild_v1.CloudBuildClient()
-    # Assuming the build source uses the path to the cloudbuild.yaml if specified.
-    source = cloudbuild_v1.RepoSource(
-        project_id=args.project_id, 
-        repo_name=args.repo_name, 
-        branch_name=args.branch_name,
-        substitutions=args.substitutions,
-        dir=args.config_path  # Path to directory containing cloudbuild.yaml
-    )
-    request = cloudbuild_v1.RunBuildTriggerRequest(
-        project_id=args.project_id, 
-        trigger_id=args.trigger_id, 
-        source=source
-    )
-    operation = client.run_build_trigger(request)
-    print(f"Triggered build with id: {operation.metadata.build.id}")
-
-def setup_deploy_subparser(subparsers):
-    """
-    Sets up an argparse subparser for the 'deploy' command.
-
-    Example command:
-    ```bash
-    sunholo deploy --project_id "my-gcp-project" --trigger_id "my-trigger-id" --repo_name "my-repo"
-    ```
-    """
-    deploy_parser = subparsers.add_parser('deploy', help='Triggers a deployment using an existing Google Cloud Build trigger.')
-    deploy_parser.add_argument('--project_id', required=True, help='Google Cloud Project ID required for deployment.')
-    deploy_parser.add_argument('--trigger_id', required=True, help='Google Cloud Build Trigger ID required for deployment.')
-    deploy_parser.add_argument('--repo_name', required=True, help='Name of the linked repository in Google Cloud Source Repositories required for deployment.')
-    deploy_parser.add_argument('--branch_name', default='dev', help='Branch name to trigger the build from, defaults to "dev".')
-    deploy_parser.add_argument('--config_path', default='.', help='Path to the directory containing the cloudbuild.yaml file, defaults to current directory.')
-    deploy_parser.set_defaults(func=trigger_build)
+from .configs import setup_list_configs_subparser
+from .deploy import setup_deploy_subparser
+from .cli_init import setup_init_subparser
 
 def main(args=None):
     """
@@ -62,14 +11,22 @@ def main(args=None):
 
     Example commands:
     ```bash
-    sunholo deploy --vac edmonbrain --config=llm_config.yaml
+    sunholo deploy --config_path . --gcs_bucket your-gcs-bucket --lancedb_bucket your-lancedb-bucket
     ```
     """
-    parser = argparse.ArgumentParser(description="sunholo CLI tool for deploying applications using Google Cloud Build.")
-    subparsers = parser.add_subparsers(title='commands', description='Valid commands', help='`sunholo deploy --help`', dest='command', required=True)
+    parser = argparse.ArgumentParser(description="sunholo CLI tool for deploying GenAI VACs")
+    subparsers = parser.add_subparsers(title='commands', 
+                                       description='Valid commands', 
+                                       help='Commands', 
+                                       dest='command', 
+                                       required=True)
 
     # Setup deploy command
     setup_deploy_subparser(subparsers)
+    # Setup list-configs command
+    setup_list_configs_subparser(subparsers)
+    # init
+    setup_init_subparser(subparsers)
 
     args = parser.parse_args(args)
 

sunholo/cli/cli_init.py

@@ -0,0 +1,80 @@
+import os
+import shutil
+from ..utils.config import get_module_filepath
+
+def init_project(args):
+    """
+    Initializes a new sunholo project with a basic configuration file and directory structure.
+
+**Explanation:**
+
+1. **Import Necessary Modules:**
+   - `os` for file system operations.
+   - `shutil` for copying files and directories.
+   - `log` from `sunholo.logging` for logging messages.
+   - `get_module_filepath` from `sunholo.utils.config` to get the absolute path of template files.
+
+2. **`init_project` Function:**
+   - Takes an `args` object from argparse, containing the `project_name`.
+   - Creates the project directory using `os.makedirs`.
+   - Copies template files from the `templates/project` directory to the new project directory using `shutil.copy` and `shutil.copytree`.
+   - Logs informative messages about the initialization process.
+
+3. **`setup_init_subparser` Function:**
+   - Sets up the `init` subcommand for the `sunholo` CLI.
+   - Adds an argument `project_name` to specify the name of the new project.
+   - Sets the `func` attribute to `init_project`, so the parser knows which function to call when the `init` command is used.
+
+**Template Files (`templates/project`):**
+
+You'll need to create a `templates/project` directory within your `sunholo` package and place the following template files in it:
+
+* **`config/llm_config.yaml`:** A basic configuration file with placeholders for LLM settings, vector stores, etc.
+* **`config/cloud_run_urls.json`:** A template for Cloud Run URLs.
+* **`app.py`:** A basic Flask app that can be customized for the project.
+* **`.gitignore`:** A gitignore file to exclude unnecessary files from version control.
+* **`README.md`:** A README file with instructions for setting up and running the project.
+
+**Usage:**
+
+After adding this code to your `cli.py` and creating the template files, users can initialize a new project using the following command:
+
+```bash
+sunholo init my_genai_project
+```
+
+This will create a new directory named `my_genai_project` with the template files, allowing users to start building their GenAI application.
+
+    """
+    project_name = args.project_name
+    project_dir = os.path.join(os.getcwd(), project_name)
+
+    print(f"Initializing project: {project_name} in directory: {project_dir}")
+
+    # Create project directory
+    if os.path.exists(project_dir):
+        print(f"Directory {project_dir} already exists. Please choose a different project name.")
+        return
+
+    os.makedirs(project_dir)
+
+    # Copy template files
+    template_dir = get_module_filepath("templates/project")
+    for filename in os.listdir(template_dir):
+        src_path = os.path.join(template_dir, filename)
+        dest_path = os.path.join(project_dir, filename)
+        if os.path.isfile(src_path):
+            shutil.copy(src_path, dest_path)
+        elif os.path.isdir(src_path):
+            shutil.copytree(src_path, dest_path)
+
+    print(f"Project {project_name} initialized successfully.")
+    print(f"Navigate to {project_dir} and customize the configuration files in the 'config' directory.")
+
+def setup_init_subparser(subparsers):
+    """
+    Sets up an argparse subparser for the 'init' command.
+    """
+    init_parser = subparsers.add_parser('init', help='Initializes a new sunholo project.')
+    init_parser.add_argument('project_name', help='The name of the new project.')
+    init_parser.set_defaults(func=init_project)

sunholo/cli/configs.py

@@ -0,0 +1,29 @@
+from ..utils.config import load_all_configs
+
+from pprint import pprint
+
+def list_configs(args):
+    """
+    Lists configuration files, filtered by kind if specified.
+    """
+    print("Listing configuration files")
+    configs = load_all_configs()
+
+    if args.kind:
+        if args.kind in configs:
+            print(f"## Config kind: {args.kind}")
+            pprint(configs[args.kind])
+        else:
+            print(f"No configurations found for kind: {args.kind}")
+    else:
+        for kind, config in configs.items():
+            pprint(f"## Config kind: {kind}") 
+            pprint(config)
+
+def setup_list_configs_subparser(subparsers):
+    """
+    Sets up an argparse subparser for the 'list-configs' command.
+    """
+    list_configs_parser = subparsers.add_parser('list-configs', help='Lists all configuration files and their details.')
+    list_configs_parser.add_argument('--kind', help='Filter configurations by kind.')
+    list_configs_parser.set_defaults(func=list_configs)

sunholo/cli/deploy.py

@@ -0,0 +1,43 @@
+import os
+from subprocess import Popen
+from ..utils.config import load_all_configs
+
+def deploy_vac(args):
+    """
+    Deploys the VAC by running a Flask app locally.
+    """
+    print(f"Deploying VAC: {args.vac_name} locally")
+
+    # Load the vacConfig
+    configs_by_kind = load_all_configs()
+    vac_config = configs_by_kind.get('vacConfig', {}).get('vac', {}).get(args.vac_name)
+
+    if not vac_config:
+        raise ValueError(f"No configuration found for VAC: {args.vac_name}")
+
+    # Assuming the Flask app is in 'app.py' within the config path
+    app_path = os.path.join(args.config_path, 'app.py')
+    if not os.path.exists(app_path):
+        raise ValueError(f"app.py not found in {args.config_path}")
+
+    print(f"Running Flask app from {app_path}")
+
+    # Run the Flask app
+    command = ["python", app_path]
+    print(f"Running Flask app with command: {' '.join(command)}")
+    process = Popen(command)
+    process.communicate()
+
+def setup_deploy_subparser(subparsers):
+    """
+    Sets up an argparse subparser for the 'deploy' command.
+
+    Example command:
+    ```bash
+    sunholo deploy "vac_name" --config_path .
+    ```
+    """
+    deploy_parser = subparsers.add_parser('deploy', help='Triggers a deployment of a VAC.')
+    deploy_parser.add_argument('vac_name', help='The name of the VAC to deploy.')
+    deploy_parser.add_argument('--config_path', default='.', help='Path to the directory containing the config folder `config/` and Flask app `app.py`, defaults to current directory.  Set _CONFIG_FOLDER env var to change config location.')
+    deploy_parser.set_defaults(func=deploy_vac)

sunholo/components/retriever.py

@@ -27,7 +27,7 @@ from langchain.retrievers import ContextualCompressionRetriever
 
 
 def load_memories(vector_name):
-    memories = load_config_key("memory", vector_name, filename="config/llm_config.yaml")
+    memories = load_config_key("memory", vector_name, type="vacConfig")
     log.info(f"Found memory settings for {vector_name}: {memories}")
     if len(memories) == 0:
         log.info(f"No memory settings found for {vector_name}")
@@ -49,7 +49,8 @@ def pick_retriever(vector_name, embeddings=None):
                 if embeddings is None:
                     embeddings = get_embeddings(vector_name)
                 vectorstore = pick_vectorstore(vectorstore, vector_name=vector_name, embeddings=embeddings)
-                vs_retriever = vectorstore.as_retriever(search_kwargs=dict(k=3))
+                k_override = value.get('k', 3)
+                vs_retriever = vectorstore.as_retriever(search_kwargs=dict(k=k_override))
                 retriever_list.append(vs_retriever)
             
             if value.get('provider') == "GoogleCloudEnterpriseSearchRetriever":
@@ -68,6 +69,10 @@ def pick_retriever(vector_name, embeddings=None):
     if len(retriever_list) == 0:
         log.info(f"No retrievers were created for {memories}")
         return None
+    
+    k_override = load_config_key("memory_k", vector_name, type="vacConfig")
+    if not k_override:
+        k_override = 3
         
     lotr = MergerRetriever(retrievers=retriever_list)
 
@@ -76,6 +81,6 @@ def pick_retriever(vector_name, embeddings=None):
     pipeline = DocumentCompressorPipeline(transformers=[filter])
     retriever = ContextualCompressionRetriever(
         base_compressor=pipeline, base_retriever=lotr, 
-        k=3)
+        k=k_override)
 
     return retriever

sunholo/embedder/embed_chunk.py

@@ -115,6 +115,15 @@ def embed_pubsub_chunk(data: dict):
                 embed_llm = value.get('llm')
                 if embed_llm:
                     embeddings = pick_embedding(embed_llm)
+                # check if read only
+                read_only = value.get('readonly')
+                if read_only:
+                    continue
+                # read from a different vector_name
+                vector_name_other = value.get('vector_name')
+                if vector_name_other:
+                    log.warning(f"Using different vector_name for vectorstore: {vector_name_other} overriding {vector_name}")
+                    vector_name = vector_name_other
                 vectorstore_obj = pick_vectorstore(vectorstore, vector_name=vector_name, embeddings=embeddings)
                 vs_retriever = vectorstore_obj.as_retriever(search_kwargs=dict(k=3))
                 vectorstore_list.append(vs_retriever)

sunholo/llamaindex/import_files.py

@@ -1,54 +1,14 @@
-
 try:
     from vertexai.preview import rag
-    from vertexai.preview.generative_models import GenerativeModel, Tool
-    import vertexai
 except ImportError:
     rag = None
 
 from ..logging import log
 from ..utils.config import load_config_key
+from ..vertex import init_vertex
 
 # Create a RAG Corpus, Import Files
 
-def init_vertex(gcp_config):
-    """
-    Initializes the Vertex AI environment using the provided Google Cloud Platform configuration.
-
-    This function configures the Vertex AI API session with specified project and location details
-    from the gcp_config dictionary. It is essential to call this function at the beginning of a session
-    before performing any operations related to Vertex AI.
-
-    Parameters:
-        gcp_config (dict): A dictionary containing the Google Cloud Platform configuration with keys:
-            - 'project_id': The Google Cloud project ID to configure for Vertex AI.
-            - 'location': The Google Cloud region to configure for Vertex AI.
-
-    Raises:
-        KeyError: If the necessary keys ('project_id' or 'location') are missing in the gcp_config dictionary.
-        ModuleNotFoundError: If the Vertex AI module is not installed and needs to be installed via pip.
-
-    Example:
-    ```python
-    gcp_config = {
-         'project_id': 'your-project-id',
-         'location': 'us-central1'
-    }
-    init_vertex(gcp_config)
-    # This will initialize the Vertex AI session with the provided project ID and location.
-
-    Note:
-        Ensure that the 'vertexai' module is installed and correctly configured before calling this function.
-        The function assumes that the required 'vertexai' library is available and that the logging setup is already in place.
-    """
-    if not rag:
-        log.error("Need to install vertexai module via `pip install google-cloud-aiplatform`")
-    
-    # Initialize Vertex AI API once per session
-    project_id = gcp_config.get('project_id')
-    location = gcp_config.get('location')
-    vertexai.init(project=project_id, location=location)
-
 def get_corpus(gcp_config):
     """
     Retrieves a LlamaIndex corpus from Vertex AI based on the provided Google Cloud configuration.
@@ -86,6 +46,9 @@ def get_corpus(gcp_config):
         print("Error fetching corpus:", str(e))
     ```
     """
+    if not rag:
+        raise ValueError("Need to install vertexai module via `pip install sunholo[gcp]`")
+    
     project_id = gcp_config.get('project_id')
     location = gcp_config.get('location')
     rag_id = gcp_config.get('rag_id')
@@ -136,7 +99,10 @@ def do_llamaindex(message_data, metadata, vector_name):
     # Imported file to corpus: {'status': 'success'}
     ```
     """
-    gcp_config = load_config_key("gcp_config", vector_name=vector_name, filename = "config/llm_config.yaml")
+    if not rag:
+        raise ValueError("Need to install vertexai module via `pip install sunholo[gcp]`")
+
+    gcp_config = load_config_key("gcp_config", vector_name=vector_name, type="vacConfig")
     if not gcp_config:
         raise ValueError(f"Need config.{vector_name}.gcp_config to configure llamaindex on VertexAI")
 
@@ -154,7 +120,7 @@ def do_llamaindex(message_data, metadata, vector_name):
     log.info(f"Found llamaindex corpus: {corpus}")
 
     # native support for cloud storage and drive links
-    chunker_config = load_config_key("chunker", vector_name=vector_name, filename="config/llm_config.yaml")
+    chunker_config = load_config_key("chunker", vector_name=vector_name, type="vacConfig")
 
     if message_data.startswith("gs://") or message_data.startswith("https://drive.google.com"):
         log.info(f"rag.import_files for {message_data}")
@@ -193,12 +159,8 @@ def do_llamaindex(message_data, metadata, vector_name):
         #    description=description,
         #)
 
-
-def llamaindex_chunker_check(message_data, metadata, vector_name):
-    # llamaindex handles its own chunking/embedding
-    memories = load_config_key("memory", vector_name=vector_name, filename = "config/llm_config.yaml")
-    total_memories = len(memories)
-    llama = None
+def check_llamaindex_in_memory(vector_name):
+    memories = load_config_key("memory", vector_name=vector_name, type="vacConfig")
     for memory in memories:  # Iterate over the list
         for key, value in memory.items():  # Now iterate over the dictionary
             log.info(f"Found memory {key}")
@@ -206,10 +168,19 @@ def llamaindex_chunker_check(message_data, metadata, vector_name):
             if vectorstore:
                 log.info(f"Found vectorstore {vectorstore}")
                 if vectorstore == "llamaindex":
-                    # https://cloud.google.com/vertex-ai/generative-ai/docs/llamaindex-on-vertexai
-                    log.info(f"llamaindex on vertex indexing for {vector_name}")
-                    llama = do_llamaindex(message_data, metadata, vector_name)
-                    log.info(f"Processed llamaindex: {llama}")
+
+                    return True
+    
+    return False
+
+def llamaindex_chunker_check(message_data, metadata, vector_name):
+    # llamaindex handles its own chunking/embedding
+    memories = load_config_key("memory", vector_name=vector_name, type="vacConfig")
+    total_memories = len(memories)
+    llama = None
+    if check_llamaindex_in_memory(vector_name):
+        llama = do_llamaindex(message_data, metadata, vector_name)
+        log.info(f"Processed llamaindex: {llama}")
 
     # If llamaindex is the only entry, return
     if llama and total_memories == 1:

sunholo/logging.py

@@ -249,4 +249,12 @@ def log_folder_location(folder_name):
     else:
         logging.warning(f"The folder '{folder_name}' does not exist in the current working directory: {current_working_directory}")
 
-log = setup_logging("sunholo")
+# lazy eval
+_logger = None
+def get_logger():
+    global _logger
+    if _logger is None:
+        _logger = setup_logging("sunholo")
+    return _logger
+
+log = get_logger()

sunholo/utils/big_context.py

@@ -0,0 +1,144 @@
+import os
+
+def has_text_extension(file_path):
+    """
+    Check if a file has a common text or code file extension.
+
+    Args:
+        file_path (str): The path to the file.
+
+    Returns:
+        bool: True if the file has a text extension, False otherwise.
+    
+    Examples:
+        >>> has_text_extension("example.txt")
+        True
+        >>> has_text_extension("example.pdf")
+        False
+    """
+    # Define a set of common text and code file extensions
+    text_extensions = {
+        '.txt', '.md', '.py', '.json', '.xml', '.csv', '.html', '.htm',
+        '.css', '.js', '.java', '.c', '.cpp', '.h', '.hpp', '.r', '.sh',
+        '.bat', '.ini', '.yaml', '.yml', '.toml', '.pl', '.rb', '.go', 
+        '.ts', '.tsx', '.rs', '.swift', '.kt', '.kts', '.scala', '.sql'
+    }
+    # Get the file extension and check if it's in the set
+    _, ext = os.path.splitext(file_path)
+    return ext.lower() in text_extensions
+
+def load_gitignore_patterns(gitignore_path):
+    """
+    Load .gitignore file and compile ignore patterns.
+
+    Args:
+        gitignore_path (str): The path to the .gitignore file.
+
+    Returns:
+        list: A list of patterns to ignore.
+    
+    Examples:
+        >>> patterns = load_gitignore_patterns("path/to/source/folder/.gitignore")
+    """
+    with open(gitignore_path, 'r') as f:
+        patterns = [line.strip() for line in f if line.strip() and not line.startswith('#')]
+    return patterns
+
+def should_ignore(file_path, patterns):
+    """
+    Check if a file path matches any of the ignore patterns.
+
+    Args:
+        file_path (str): The path to the file.
+        patterns (list): A list of patterns to ignore.
+
+    Returns:
+        bool: True if the file should be ignored, False otherwise.
+    
+    Examples:
+        >>> should_ignore("path/to/file.txt", ["*.txt", "node_modules/"])
+        True
+    """
+    from fnmatch import fnmatch
+    rel_path = os.path.relpath(file_path)
+    for pattern in patterns:
+        if fnmatch(rel_path, pattern) or fnmatch(os.path.basename(rel_path), pattern):
+            return True
+    return False
+
+def build_file_tree(source_folder, patterns):
+    """
+    Build a hierarchical file tree structure of a directory, ignoring files and directories in .gitignore.
+
+    Args:
+        source_folder (str): The root directory to build the file tree from.
+        patterns (list): A list of patterns to ignore.
+
+    Returns:
+        list: A list of strings representing the file tree structure.
+    
+    Examples:
+        >>> build_file_tree("path/to/source/folder", patterns)
+        ['source_folder/', '    file1.txt', '    subfolder/', '        file2.py']
+    """
+    file_tree = []
+    for root, dirs, files in os.walk(source_folder):
+        # Filter out ignored directories
+        dirs[:] = [d for d in dirs if not should_ignore(os.path.join(root, d), patterns)]
+        # Filter out ignored files
+        files = [f for f in files if not should_ignore(os.path.join(root, f), patterns)]
+        
+        level = root.replace(source_folder, '').count(os.sep)
+        indent = ' ' * 4 * (level)
+        file_tree.append(f"{indent}{os.path.basename(root)}/")
+        sub_indent = ' ' * 4 * (level + 1)
+        for f in files:
+            file_tree.append(f"{sub_indent}{f}")
+    return file_tree
+
+def merge_text_files(source_folder, output_file, patterns):
+    """
+    Merge the contents of all readable text files in a directory into one file.
+    Also append the file tree structure at the end of the output file.
+
+    Args:
+        source_folder (str): The directory containing the text files to merge.
+        output_file (str): The path to the output file where contents will be written.
+        patterns (list): A list of patterns to ignore.
+
+    Examples:
+        >>> merge_text_files("path/to/source/folder", "path/to/output/bigfile.txt", patterns)
+    """
+    file_tree = build_file_tree(source_folder, patterns)
+    with open(output_file, 'w', encoding='utf-8') as outfile:
+        for root, dirs, files in os.walk(source_folder):
+            # Filter out ignored directories
+            dirs[:] = [d for d in dirs if not should_ignore(os.path.join(root, d), patterns)]
+            # Filter out ignored files
+            files = [f for f in files if not should_ignore(os.path.join(root, f), patterns)]
+            
+            for file_name in files:
+                file_path = os.path.join(root, file_name)
+                if has_text_extension(file_path):
+                    try:
+                        with open(file_path, 'r', encoding='utf-8') as infile:
+                            outfile.write(f"--- Start of {file_path} ---\n")
+                            outfile.write(infile.read())
+                            outfile.write(f"\n--- End of {file_path} ---\n\n")
+                    except (IOError, UnicodeDecodeError):
+                        print(f"Skipping file (cannot read as text): {file_path}")
+        outfile.write("\n--- File Tree ---\n")
+        outfile.write("\n".join(file_tree))
+
+# Example usage
+if __name__ == "__main__":
+    source_folder = 'sunholo'
+    output_file = 'bigfile.txt'
+    gitignore_path = os.path.join(source_folder, '.gitignore')
+    
+    if os.path.exists(gitignore_path):
+        patterns = load_gitignore_patterns(gitignore_path)
+    else:
+        patterns = []  # Empty list if no .gitignore
+
+    merge_text_files(source_folder, output_file, patterns)

sunholo/utils/config.py

@@ -65,6 +65,9 @@ def load_all_configs():
     configs_by_kind = defaultdict(dict)
     for filename in os.listdir(config_folder):
         log.info(f"config file: {filename}")
+        if filename in ["cloudbuild.yaml", "cloud_run_urls.json"]:
+            # skip these
+            continue
         if filename.endswith(('.yaml', '.yml', '.json')):
             config_file = os.path.join(config_folder, filename)
             

sunholo/vertex/init_vertex.py

@@ -0,0 +1,43 @@
+from ..logging import log
+
+def init_vertex(gcp_config):
+    """
+    Initializes the Vertex AI environment using the provided Google Cloud Platform configuration.
+
+    This function configures the Vertex AI API session with specified project and location details
+    from the gcp_config dictionary. It is essential to call this function at the beginning of a session
+    before performing any operations related to Vertex AI.
+
+    Parameters:
+        gcp_config (dict): A dictionary containing the Google Cloud Platform configuration with keys:
+            - 'project_id': The Google Cloud project ID to configure for Vertex AI.
+            - 'location': The Google Cloud region to configure for Vertex AI.
+
+    Raises:
+        KeyError: If the necessary keys ('project_id' or 'location') are missing in the gcp_config dictionary.
+        ModuleNotFoundError: If the Vertex AI module is not installed and needs to be installed via pip.
+
+    Example:
+    ```python
+    gcp_config = {
+         'project_id': 'your-project-id',
+         'location': 'us-central1'
+    }
+    init_vertex(gcp_config)
+    # This will initialize the Vertex AI session with the provided project ID and location.
+
+    Note:
+        Ensure that the 'vertexai' module is installed and correctly configured before calling this function.
+        The function assumes that the required 'vertexai' library is available and that the logging setup is already in place.
+    """
+    try:
+        import vertexai
+    except ImportError:
+        log.error("Need to install vertexai module via `pip install sunholo[gcp]`")
+
+        return None
+    
+    # Initialize Vertex AI API once per session
+    project_id = gcp_config.get('project_id')
+    location = gcp_config.get('location')
+    vertexai.init(project=project_id, location=location)

{sunholo-0.57.2.dist-info → sunholo-0.58.2.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.57.2
+Version: 0.58.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.57.2.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.58.2.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0
@@ -24,13 +24,19 @@ Requires-Dist: langchain-community
 Provides-Extra: all
 Requires-Dist: asyncpg ; extra == 'all'
 Requires-Dist: flask ; extra == 'all'
+Requires-Dist: google-auth ; extra == 'all'
+Requires-Dist: google-auth-httplib2 ; extra == 'all'
+Requires-Dist: google-auth-oauthlib ; extra == 'all'
 Requires-Dist: google-cloud-aiplatform ; extra == 'all'
 Requires-Dist: google-api-python-client ; extra == 'all'
 Requires-Dist: google-cloud-alloydb-connector[pg8000] ; extra == 'all'
+Requires-Dist: google-cloud-bigquery ; extra == 'all'
 Requires-Dist: google-cloud-build ; extra == 'all'
 Requires-Dist: google-cloud-logging ; extra == 'all'
 Requires-Dist: google-cloud-storage ; extra == 'all'
 Requires-Dist: google-cloud-pubsub ; extra == 'all'
+Requires-Dist: google-cloud-discoveryengine ; extra == 'all'
+Requires-Dist: google-generativeai ; extra == 'all'
 Requires-Dist: gunicorn ; extra == 'all'
 Requires-Dist: httpcore ; extra == 'all'
 Requires-Dist: httpx ; extra == 'all'
@@ -44,21 +50,35 @@ Requires-Dist: langchain-google-alloydb-pg ; extra == 'all'
 Requires-Dist: langchain-anthropic ; extra == 'all'
 Requires-Dist: langfuse ; extra == 'all'
 Requires-Dist: pg8000 ; extra == 'all'
+Requires-Dist: pgvector ; extra == 'all'
+Requires-Dist: psycopg2-binary ; extra == 'all'
+Requires-Dist: pypdf ; extra == 'all'
 Requires-Dist: fastapi ; extra == 'all'
+Requires-Dist: supabase ; extra == 'all'
+Requires-Dist: tiktoken ; extra == 'all'
 Requires-Dist: python-socketio ; extra == 'all'
 Provides-Extra: anthropic
 Requires-Dist: langchain-anthropic ; extra == 'anthropic'
 Provides-Extra: database
 Requires-Dist: asyncpg ; extra == 'database'
+Requires-Dist: supabase ; extra == 'database'
 Requires-Dist: sqlalchemy ; extra == 'database'
 Requires-Dist: pg8000 ; extra == 'database'
+Requires-Dist: pgvector ; extra == 'database'
+Requires-Dist: psycopg2-binary ; extra == 'database'
 Requires-Dist: lancedb ; extra == 'database'
 Provides-Extra: gcp
+Requires-Dist: google-auth ; extra == 'gcp'
+Requires-Dist: google-auth-httplib2 ; extra == 'gcp'
+Requires-Dist: google-auth-oauthlib ; extra == 'gcp'
 Requires-Dist: google-cloud-aiplatform ; extra == 'gcp'
+Requires-Dist: google-cloud-bigquery ; extra == 'gcp'
 Requires-Dist: google-cloud-build ; extra == 'gcp'
 Requires-Dist: google-cloud-storage ; extra == 'gcp'
 Requires-Dist: google-cloud-logging ; extra == 'gcp'
 Requires-Dist: google-cloud-pubsub ; extra == 'gcp'
+Requires-Dist: google-cloud-discoveryengine ; extra == 'gcp'
+Requires-Dist: google-generativeai ; extra == 'gcp'
 Requires-Dist: langchain-google-genai ; extra == 'gcp'
 Requires-Dist: langchain-google-alloydb-pg ; extra == 'gcp'
 Requires-Dist: google-api-python-client ; extra == 'gcp'
@@ -71,8 +91,15 @@ Requires-Dist: httpcore ; extra == 'http'
 Requires-Dist: httpx ; extra == 'http'
 Requires-Dist: langfuse ; extra == 'http'
 Requires-Dist: python-socketio ; extra == 'http'
+Requires-Dist: requests ; extra == 'http'
 Provides-Extra: openai
 Requires-Dist: langchain-openai ; extra == 'openai'
+Requires-Dist: tiktoken ; extra == 'openai'
+Provides-Extra: pipeline
+Requires-Dist: GitPython ; extra == 'pipeline'
+Requires-Dist: lark ; extra == 'pipeline'
+Requires-Dist: pypdf ; extra == 'pipeline'
+Requires-Dist: tabulate ; extra == 'pipeline'
 
 ## Introduction
 This is the Sunholo Python project, a comprehensive toolkit for working with language models and vector stores on Google Cloud Platform. It provides a wide range of functionalities and utilities to facilitate the development and deployment of language model applications.

{sunholo-0.57.2.dist-info → sunholo-0.58.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
-sunholo/__init__.py,sha256=n5NRH5j9yi04KFNp1t8-lBQYWxSW01c2LGB1R90iGeM,820
-sunholo/logging.py,sha256=_poZ61TbLl6yHox2eO78BvIGQCUCrAIjWio70sGJIag,11233
+sunholo/__init__.py,sha256=KfqJV0T-2WCIlabNxeX3CvLS-bBGCl9n5aWy091lr2M,841
+sunholo/logging.py,sha256=JSzS_6HY6BIxy8bmVV2TDHXOjyfB1q5nI76OZXAtQDc,11372
 sunholo/agents/__init__.py,sha256=CnlbVohPt-Doth9PyROSlN3P8xMV9j9yS19YE-wCS90,341
 sunholo/agents/chat_history.py,sha256=PbwYmw1TwzI8H-cwQIGgHZ6UIr2Qb-JWow0RG3ayLM8,5195
 sunholo/agents/dispatch_to_qa.py,sha256=kWrO-CJel5kJAyyCShShpACUuZpqDOP7DN8vo_7ciao,8056
@@ -13,7 +13,7 @@ sunholo/agents/fastapi/base.py,sha256=clk76cHbUAvU0OYJrRfCWX_5f0ACbhDsIzYBhI3wyo
 sunholo/agents/fastapi/qna_routes.py,sha256=DgK4Btu5XriOC1JaRQ4G_nWEjJfnQ0J5pyLanF6eF1g,3857
 sunholo/agents/flask/__init__.py,sha256=uqfHNw2Ru3EJ4dJEcbp86h_lkquBQPMxZbjhV_xe3rs,72
 sunholo/agents/flask/base.py,sha256=RUGWBYWeV60FatYF5sMRrxD-INU97Vodsi6JaB6i93s,763
-sunholo/agents/flask/qna_routes.py,sha256=gg041eCoTpjr7ZuPPJb5z2KlIIbh85Wc4-a6bjGLiHs,8573
+sunholo/agents/flask/qna_routes.py,sha256=JLjYrVN2mGoWrGUM_o5N93um46gXpQa8LWdxjF2yN4Y,8554
 sunholo/archive/__init__.py,sha256=qNHWm5rGPVOlxZBZCpA1wTYPbalizRT7f8X4rs2t290,31
 sunholo/archive/archive.py,sha256=C-UhG5x-XtZ8VheQp92IYJqgD0V3NFQjniqlit94t18,1197
 sunholo/auth/__init__.py,sha256=4owDjSaWYkbTlPK47UHTOC0gCWbZsqn4ZIEw5NWZTlg,28
@@ -22,7 +22,7 @@ sunholo/bots/__init__.py,sha256=EMFd7e2z68l6pzYOnkzHbLd2xJRvxTKFRNCTuhZ8hIw,130
 sunholo/bots/discord.py,sha256=cCFae5K1BCa6JVkWGLh_iZ9qFO1JpXb6K4eJrlDfEro,2442
 sunholo/bots/webapp.py,sha256=EIMxdAJ_xtufwJmvnn7N_Fb_1hZ9DjhJ0Kf_hp02vEU,1926
 sunholo/chunker/__init__.py,sha256=UhQBZTKwDfBXm0TPv4LvsGc5pdUGCbYzi3qPTOkU4gw,55
-sunholo/chunker/data_to_embed_pubsub.py,sha256=9saAZDFWG7DTtlL1p2ZfB9Fge9jWPjfVyRa1OnuonPk,2899
+sunholo/chunker/data_to_embed_pubsub.py,sha256=t-pWNYv2mnwVAkMcIOK2CrIb3yr2aS9iAdtryk7hT8o,2931
 sunholo/chunker/doc_handling.py,sha256=2w5oDkU2RX3ynq7GkR5CUouiEzBXbEkaK_4p6yRcC3M,8597
 sunholo/chunker/images.py,sha256=Xmh1vwHrVhoXm5iH2dhCc52O8YgdzE8KrDSdL-pGnp8,1861
 sunholo/chunker/loaders.py,sha256=xiToUVgPz2ZzcqpUAq7aNP3PTenb_rBUAFzu0JPycIg,10268
@@ -31,11 +31,14 @@ sunholo/chunker/pdfs.py,sha256=daCZ1xjn1YvxlifIyxskWNpLJLe-Q9D_Jq12MWx3tZo,2473
 sunholo/chunker/publish.py,sha256=PoT8q3XJeFCg10WrLkYhuaaXIrGVkvUD3-R9IfoWoH4,2703
 sunholo/chunker/splitter.py,sha256=ug_v-h0wos3b7OkhmedVQs5jtLuDdFDWypvsZVYgxbU,6743
 sunholo/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sunholo/cli/cli.py,sha256=9ov0GgbdM3IDC4QgqLnG4EBSiCm2_8MlRS-6G45sXJg,3279
+sunholo/cli/cli.py,sha256=rcO1hMthy5nWC_5sOHqRm7ut70c9JfxFTSjFRBNYuYg,1248
+sunholo/cli/cli_init.py,sha256=WReZuMQwDfkRUvssYT7TirUoG6SiT1dTDol8nLI8O70,3418
+sunholo/cli/configs.py,sha256=wLRSL-Z58B7dQ_gxTK_d8HjNP5pABhegLqjB0-IKGjg,992
+sunholo/cli/deploy.py,sha256=zxdwUsRTRMC8U5vyRv0JiKBLFn84Ug_Tc88-_h9hJSs,1609
 sunholo/components/__init__.py,sha256=RJGNEihwvRIiDScKis04RHJv4yZGI1UpXlOmuCptNZI,208
 sunholo/components/llm.py,sha256=T4we3tGmqUj4tPwxQr9M6AXv_BALqZV_dRSvINan-oU,10374
 sunholo/components/prompt.py,sha256=eZSghXkIlRzXiSrzgkG7e5ytUYq6R6LV-qjHU8jStig,6353
-sunholo/components/retriever.py,sha256=TiM-axCeaZ6CZ8rGKGx-io16JKDe8z0pnMccBi1yqHw,3509
+sunholo/components/retriever.py,sha256=QA_l7HXwd4g6IFuT2A5mICzUcV80K0YvDbCtRxefq8o,3684
 sunholo/components/vectorstore.py,sha256=RB_Dgc9234G_TE3w3abCfBw1pqm2br2RrLP0UqshkvA,5172
 sunholo/database/__init__.py,sha256=Zz0Shcq-CtStf9rJGIYB_Ybzb8rY_Q9mfSj-nviM490,241
 sunholo/database/alloydb.py,sha256=18Q4AG_W-Sz8udIj3gWMkGrMWmiEelqgOwJ7VKHElV0,14877
@@ -50,7 +53,7 @@ sunholo/database/sql/sb/delete_source_row.sql,sha256=r6fEuUKdbiLHCDGKSbKINDCpJjs
 sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUtYL5cCxbC2mj_c,255
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
-sunholo/embedder/embed_chunk.py,sha256=8kZI2XB1uB6joCqzVFjelqV3cXuqc6iog_SuFKR99x8,5376
+sunholo/embedder/embed_chunk.py,sha256=XV1kdDUWw2QO-am5_Yl7GrYP9V_4i1XRNNFPhqUSnZQ,5851
 sunholo/gcs/__init__.py,sha256=DtVw_AZwQn-IguR5BJuIi2XJeF_FQXizhJikzRNrXiE,50
 sunholo/gcs/add_file.py,sha256=JmJIuz5Z1h7-eJ6s2eE3wc8Y4IAv3Jridq1xfQbD9_E,4711
 sunholo/gcs/download_url.py,sha256=PAwYShV-sRd9sNvuJrEOvfF1V34ovVP0omWbuwDkRrA,4751
@@ -60,7 +63,7 @@ sunholo/langfuse/callback.py,sha256=G9xcZHpLvyzolU57ycItLaooMCtRuM37QJSWjiwQEd0,
 sunholo/langfuse/prompts.py,sha256=HO4Zy9usn5tKooBPCKksuw4Lff3c03Ny5wqn4ce_xZM,1217
 sunholo/llamaindex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/llamaindex/generate.py,sha256=l1Picr-hVwkmAUD7XmTCa63qY9ERliFHQXwyX3BqB2Q,686
-sunholo/llamaindex/import_files.py,sha256=QamEVtFXSD6Zk2La0HDgmtVPhFDdXpjzLACO--BAYpg,9378
+sunholo/llamaindex/import_files.py,sha256=j71_rGIpZg1Zbsy-PnNIUgpNJznnsDiYxR6sulBlQ_8,7786
 sunholo/lookup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/lookup/model_lookup.yaml,sha256=O7o-jP53MLA06C8pI-ILwERShO-xf6z_258wtpZBv6A,739
 sunholo/patches/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -80,12 +83,15 @@ sunholo/streaming/streaming.py,sha256=H8qpQC5wr4dVyInWTAjTmvGePVueWcwNaR8T_5mEp6
 sunholo/summarise/__init__.py,sha256=MZk3dblUMODcPb1crq4v-Z508NrFIpkSWNf9FIO8BcU,38
 sunholo/summarise/summarise.py,sha256=C3HhjepTjUhUC8FLk4jMQIBvq1BcORniwuTFHjPVhVo,3784
 sunholo/utils/__init__.py,sha256=G11nN_6ATjxpuMfG_BvcUr9UU8onPIgkpTK6CjOcbr8,48
-sunholo/utils/config.py,sha256=NW2FFyNNTwCyopOvSzDQ0I0l92LAfJJ7hEzatSuoZho,8689
+sunholo/utils/big_context.py,sha256=qHYtds4Ecf9eZRHVqXho4_q8Je7HD44-vS6RJ6s9Z0Q,5387
+sunholo/utils/config.py,sha256=Ve1sb68Av9_SPGqXs33g5FAJSIQ3GODoeuUCW3MNCwU,8802
 sunholo/utils/gcp.py,sha256=B2G1YKjeD7X9dqO86Jrp2vPuFwZ223Xl5Tg09Ndw-oc,5760
 sunholo/utils/parsers.py,sha256=OrHmASqIbI45atVOhiGodgLvnfrzkvVzyHnSvAXD89I,3841
-sunholo-0.57.2.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.57.2.dist-info/METADATA,sha256=JMXDwlRVc5IDqCb17ls_c_rV9kyzHsTRPnjAEY7mbAQ,6617
-sunholo-0.57.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-sunholo-0.57.2.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.57.2.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.57.2.dist-info/RECORD,,
+sunholo/vertex/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+sunholo/vertex/init_vertex.py,sha256=JDMUaBRdednzbKF-5p33qqLit2LMsvgvWW-NRz0AqO0,1801
+sunholo-0.58.2.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.58.2.dist-info/METADATA,sha256=6N_BstncHOJiJJJ4N1QOHiKDxUB_Bc8UDnB2NA_xVb0,7894
+sunholo-0.58.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+sunholo-0.58.2.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.58.2.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.58.2.dist-info/RECORD,,