sunholo 0.79.0__tar.gz → 0.79.2__tar.gz

{sunholo-0.79.0 → sunholo-0.79.2}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.79.0
+Version: 0.79.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.0.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.2.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.79.0 → sunholo-0.79.2}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 # Define your base version
-version = '0.79.0'
+version = '0.79.2'
 
 setup(
     name='sunholo',
@@ -34,7 +34,6 @@ setup(
         "langchain>=0.2.5",
         "langchain_experimental>0.0.60",
         "langchain-community",
-        # Add the minimal dependencies that your package requires here
     ],
     extras_require={
         # Define optional dependencies with feature names

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/agents/chat_history.py

@@ -1,5 +1,5 @@
 import json
-from ..logging import log
+from ..custom_logging import log
 
 def extract_chat_history(chat_history=None):
     """

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/agents/dispatch_to_qa.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 from ..utils import ConfigManager
 from ..auth import get_header
 import requests

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/agents/fastapi/base.py

@@ -11,7 +11,7 @@ try:
 except ImportError:
     FastAPI = None
 
-from ...logging import log
+from ...custom_logging import log
 
 def create_fastapi_app(origins = ["*"],
                        origin_regex = r"https://(.*\.)?sunholo\.com"):

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/agents/fastapi/qna_routes.py

@@ -15,7 +15,7 @@ from ...agents import extract_chat_history, handle_special_commands
 from ...qna.parsers import parse_output
 from ...streaming import start_streaming_chat_async
 from ...archive import archive_qa
-from ...logging import log
+from ...custom_logging import log
 
 class VACRequest(BaseModel):
     user_input: str

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/agents/flask/qna_routes.py

@@ -21,7 +21,7 @@ from ...agents import extract_chat_history, handle_special_commands
 from ...qna.parsers import parse_output
 from ...streaming import start_streaming_chat
 from ...archive import archive_qa
-from ...logging import log
+from ...custom_logging import log
 from ...utils.config import load_config
 from ...utils.version import sunholo_version
 import os

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/agents/flask/vac_routes.py

@@ -7,7 +7,7 @@ from ...agents import extract_chat_history, handle_special_commands
 from ...qna.parsers import parse_output
 from ...streaming import start_streaming_chat
 from ...archive import archive_qa
-from ...logging import log
+from ...custom_logging import log
 from ...utils.config import load_config
 from ...utils.version import sunholo_version
 import os

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/agents/langserve.py

@@ -1,5 +1,5 @@
 import requests
-from ..logging import log
+from ..custom_logging import log
 from ..auth import get_header
 
 # Global cache for storing input schemas

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/agents/pubsub.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 
 
 import base64

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/agents/route.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 from ..utils import load_config, ConfigManager
 
 def read_cloud_run_url(agent, cloud_run_urls_file='config/cloud_run_urls.json'):

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/agents/special_commands.py

@@ -26,7 +26,7 @@ from ..utils.parsers import contains_url, extract_urls
 from ..chunker.publish import publish_text
 from ..gcs.add_file import add_file_to_gcs
 from ..utils.config import load_config_key
-from ..logging import log
+from ..custom_logging import log
 
 # config file?
 command_descriptions = {

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/agents/swagger.py

@@ -1,7 +1,7 @@
 import copy
 from ..utils.config import load_all_configs
 from .route import route_vac
-from ..logging import log
+from ..custom_logging import log
 from ruamel.yaml import YAML
 from io import StringIO
 # check it here:

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/archive/archive.py

@@ -12,7 +12,7 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 from ..pubsub import PubSubManager
-from ..logging import log
+from ..custom_logging import log
 
 
 

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/auth/refresh.py

@@ -6,7 +6,7 @@ from google.auth.transport import requests
 from ..utils.gcp import is_running_on_gcp
 
 
-from ..logging import log
+from ..custom_logging import log
 
 def get_default_email():
     if not refresh_credentials():

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/auth/run.py

@@ -6,7 +6,7 @@ from ..utils.config import load_config
 from ..utils import ConfigManager
 from ..utils.gcp import is_running_on_cloudrun
 from ..utils.api_key import has_multivac_api_key, get_multivac_api_key
-from ..logging import log
+from ..custom_logging import log
 from ..agents.route import route_vac
 from .gcloud import get_local_gcloud_token
 

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/azure/auth.py

@@ -5,7 +5,7 @@ except ImportError:
     DefaultAzureCredential = None
     ClientSecretCredential = None
 
-from ..logging import log
+from ..custom_logging import log
 
 def azure_auth():
     """

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/azure/blobs.py

@@ -1,5 +1,5 @@
 import re
-from ..logging import log
+from ..custom_logging import log
 
 
 def is_azure_blob(message_data):

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/azure/event_grid.py

@@ -1,5 +1,5 @@
 # process_azure_blob_event.py
-from ..logging import log
+from ..custom_logging import log
 
 def process_azure_blob_event(events: list) -> tuple:
     """

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/bots/discord.py

@@ -13,7 +13,7 @@
 #   limitations under the License.
 import json, os
 import requests
-from ..logging import log
+from ..custom_logging import log
 
 
 

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/chunker/azure.py

@@ -14,7 +14,7 @@
 import json
 from datetime import datetime, timezone
 
-from ..logging import log
+from ..custom_logging import log
 from ..azure import process_azure_blob_event
 
 from ..invoke import invoke_vac

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/chunker/doc_handling.py

@@ -1,5 +1,5 @@
 from ..utils import load_config_key
-from ..logging import log
+from ..custom_logging import log
 from ..database.alloydb import add_document_if_not_exists
 from ..database.uuid import generate_uuid_from_object_id
 from ..components.llm import llm_str_to_llm
@@ -178,7 +178,7 @@ Be careful not to add any speculation or any details that are not covered in the
                 
                 if bucket_name.startswith("gs://"):
                     bucket_name = bucket_name[len("gs://"):]
-                    
+
                 with tempfile.NamedTemporaryFile(mode='w+', delete=False) as temp_file:
                     temp_file.write(summary)
                     temp_file.flush()

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/chunker/encode_metadata.py

@@ -3,7 +3,7 @@ import base64
 import json
 from datetime import datetime, timezone
 
-from ..logging import log
+from ..custom_logging import log
 
 def create_metadata(vac, metadata):
     now_utc = datetime.now(timezone.utc)

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/chunker/images.py

@@ -3,7 +3,7 @@ import datetime
 import tempfile
 import os
 from ..gcs.add_file import add_file_to_gcs, get_image_file_name
-from ..logging import log
+from ..custom_logging import log
 from ..utils.gcp import is_running_on_gcp
 
 

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/chunker/loaders.py

@@ -18,7 +18,7 @@ from langchain_community.document_loaders import UnstructuredURLLoader
 from langchain_community.document_loaders import GitLoader
 from langchain_community.document_loaders import GoogleDriveLoader
 
-from ..logging import log
+from ..custom_logging import log
 from .pdfs import read_pdf_file
 from ..utils.config import load_config
 

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/chunker/message_data.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 
 import pathlib
 import tempfile

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/chunker/pdfs.py

@@ -13,7 +13,7 @@
 #   limitations under the License.
 import os
 import pathlib
-from ..logging import log
+from ..custom_logging import log
 
 def split_pdf_to_pages(pdf_path, temp_dir):
 

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/chunker/process_chunker_data.py

@@ -16,7 +16,7 @@ from .publish import process_docs_chunks_vector_name
 from .splitter import chunk_doc_to_docs
 from ..azure.blobs import is_azure_blob
 
-from ..logging import log
+from ..custom_logging import log
 
 def process_chunker_data(message_data, metadata, vector_name):
 

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/chunker/publish.py

@@ -1,4 +1,4 @@
-from ..logging import log
+from ..custom_logging import log
 from ..pubsub import PubSubManager
 from ..utils.parsers import contains_url, extract_urls
 from ..utils.gcp_project import get_gcp_project

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/chunker/pubsub.py

@@ -12,7 +12,7 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-from ..logging import log
+from ..custom_logging import log
 from ..pubsub import process_pubsub_message
 from .process_chunker_data import process_chunker_data
 

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/chunker/splitter.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 from ..utils.parsers import remove_whitespace
 from langchain.schema import Document
 import langchain.text_splitter as text_splitter

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/cli/chat_vac.py

@@ -3,7 +3,7 @@ from ..streaming import generate_proxy_stream, can_agent_stream
 from ..utils.user_ids import generate_user_id
 from ..utils import ConfigManager
 from ..utils.api_key import has_multivac_api_key
-from ..logging import log
+from ..custom_logging import log
 from ..qna.parsers import parse_output
 from ..gcs.add_file import add_file_to_gcs
 from .run_proxy import clean_proxy_list, start_proxy, stop_proxy

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/cli/cli.py

@@ -10,10 +10,11 @@ from .chat_vac import setup_vac_subparser
 from .embedder import setup_embedder_subparser
 from .swagger import setup_swagger_subparser
 from .vertex import setup_vertex_subparser
+from ..llamaindex import setup_llamaindex_subparser
 
 from ..utils import ConfigManager
 
-from ..logging import log
+from ..custom_logging import log
 
 from .sun_rich import console
 import sys
@@ -87,6 +88,8 @@ def main(args=None):
     setup_swagger_subparser(subparsers)
     # vertex
     setup_vertex_subparser(subparsers)
+    # llamaindex
+    setup_llamaindex_subparser(subparsers)
 
     #TODO: add database setup commands: alloydb and supabase
 

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/components/llm.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 from ..utils import load_config_key, ConfigManager
 
 import os

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/components/retriever.py

@@ -11,9 +11,9 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 from .vectorstore import pick_vectorstore
-from ..utils import load_config_key, ConfigManager
+from ..utils import ConfigManager
 from .llm import get_embeddings
 from ..utils.gcp_project import get_gcp_project
 
@@ -41,9 +41,14 @@ def load_memories(vector_name:str=None, config:ConfigManager=None):
 
     return memories
 
-def pick_retriever(vector_name, embeddings=None):
+def pick_retriever(vector_name:str=None, config:ConfigManager=None, embeddings=None):
 
-    memories = load_memories(vector_name)
+    if config is None:
+        if vector_name is None:
+            raise ValueError("vector_name and config were none")
+        config = ConfigManager(vector_name)
+
+    memories = load_memories(config=config)
 
     retriever_list = []
     for memory in memories:  # Iterate over the list
@@ -58,7 +63,7 @@ def pick_retriever(vector_name, embeddings=None):
                     log.info(f"Skipped from_metadata_id for {vectorstore}")
                     continue
 
-                embeddings = embeddings or get_embeddings(vector_name)
+                embeddings = embeddings or get_embeddings(config=config)
                 read_only = value.get('read_only')
                 try:
                     vectorstore = pick_vectorstore(vectorstore, 
@@ -93,15 +98,15 @@ def pick_retriever(vector_name, embeddings=None):
         log.info(f"No retrievers were created for {memories}")
         return None
     
-    retriever = process_retrieval(retriever_list, vector_name)
+    retriever = process_retrieval(retriever_list, config=config)
 
     return retriever
 
-def metadata_retriever(metadata: dict, key: str, vector_name:str, embeddings=None):
+def metadata_retriever(metadata: dict, key: str, config:ConfigManager, embeddings=None):
     """
     Decides which vector_name to retrieve from metadata passed
     """
-    memories = load_memories(vector_name)
+    memories = load_memories(config=config)
 
     retriever_list = []
     for memory in memories:  # Iterate over the list
@@ -118,7 +123,7 @@ def metadata_retriever(metadata: dict, key: str, vector_name:str, embeddings=Non
                         raise ValueError(f"Missing {key} in {metadata}")
                     the_id = metadata[key]
                     read_only = value.get('read_only')
-                    embeddings = embeddings or get_embeddings(vector_name)
+                    embeddings = embeddings or get_embeddings(config=config)
                     vectorstore = pick_vectorstore(vectorstore, 
                                                    vector_name=the_id, 
                                                    embeddings=embeddings, 
@@ -133,17 +138,18 @@ def metadata_retriever(metadata: dict, key: str, vector_name:str, embeddings=Non
         log.info(f"No retrievers were created for {memories}")
         return None
     
-    retriever = process_retrieval(retriever_list, vector_name)
+    retriever = process_retrieval(retriever_list, config=config)
 
     return retriever
     
 
 
-def process_retrieval(retriever_list: list, vector_name: str):
-    k_override = load_config_key("memory_k", vector_name, kind="vacConfig")
+def process_retrieval(retriever_list: list, config: ConfigManager):
+    k_override = config.vacConfig('memory_k')
     lotr = MergerRetriever(retrievers=retriever_list)
 
-    filter_embeddings = get_embeddings(vector_name)
+    filter_embeddings = get_embeddings(config=config)
+
     filter = EmbeddingsRedundantFilter(embeddings=filter_embeddings)
     pipeline = DocumentCompressorPipeline(transformers=[filter])
     retriever = ContextualCompressionRetriever(

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/components/vectorstore.py

@@ -12,10 +12,21 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 import os
-from ..logging import log
+from ..custom_logging import log
+from ..utils import ConfigManager
 
-def pick_vectorstore(vs_str: str, vector_name: str, embeddings, read_only=None):
+def pick_vectorstore(vs_str: str, vector_name: str=None, embeddings=None, config:ConfigManager=None, read_only=None):
     log.debug('Picking vectorstore')
+
+    if not embeddings:
+        raise ValueError("Requires embeddings")
+    
+    if config is None:
+        if vector_name is None:
+            raise ValueError("vector_name and config were none")
+        config = ConfigManager(vector_name)
+        
+    vector_name = config.vector_name
         
     if vs_str == 'supabase':
         from supabase import Client, create_client

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/database/alloydb.py

@@ -11,7 +11,7 @@ except ImportError:
 from .database import get_vector_size
 from .alloydb_client import AlloyDBClient
 
-from ..logging import log
+from ..custom_logging import log
 from ..utils.config import load_config_key
 
 

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/database/alloydb_client.py

@@ -8,7 +8,7 @@ except ImportError:
     pass
 
 from .database import get_vector_size
-from ..logging import log
+from ..custom_logging import log
 
 class AlloyDBClient:
     """

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/database/database.py

@@ -17,7 +17,7 @@ import time
 import math
 
 from ..utils.config import get_module_filepath
-from ..logging import log
+from ..custom_logging import log
 from ..utils.config import load_config_key
 
 

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/database/lancedb.py

@@ -1,4 +1,4 @@
-from ..logging import log
+from ..custom_logging import log
 
 def create_lancedb_index(bucket, vector_name, num_partitions=256, num_sub_vectors=96, text_key="text"):
     try:

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/database/static_dbs.py

@@ -1,5 +1,5 @@
 import os
-from ..logging import log_folder_location
+from ..custom_logging import log_folder_location
 
 def get_db_directory(db_dir='db'):
     current_script_directory = os.path.dirname(os.path.abspath(__file__))

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/discovery_engine/chunker_handler.py

@@ -1,4 +1,4 @@
-from ..logging import log
+from ..custom_logging import log
 from ..utils.config import load_config_key
 from ..utils.gcp_project import get_gcp_project
 from ..components import load_memories

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/discovery_engine/discovery_engine_client.py

@@ -10,7 +10,7 @@ except ImportError:
     Chunk = None
     SearchResponse = None
 
-from ..logging import log
+from ..custom_logging import log
 from typing import Optional, List
 
 class DiscoveryEngineClient:

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/embedder/embed_chunk.py

@@ -20,7 +20,7 @@ import uuid
 from langchain.schema import Document
 
 from ..components import get_embeddings, pick_vectorstore, load_memories, pick_embedding
-from ..logging import log
+from ..custom_logging import log
 from ..database.uuid import generate_uuid_from_object_id
 from ..utils import ConfigManager
 

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/gcs/add_file.py

@@ -21,7 +21,7 @@ try:
 except ImportError:
     storage = None
 
-from ..logging import log
+from ..custom_logging import log
 from ..utils import load_config_key, ConfigManager
 
 

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/gcs/download_folder.py

@@ -5,7 +5,7 @@ try:
 except ImportError:
     storage = None
 
-from ..logging import log
+from ..custom_logging import log
 
 def download_files_from_gcs(bucket_name: str, source_folder: str, destination_folder: str=None):
     """

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/gcs/download_url.py

@@ -10,7 +10,7 @@ try:
 except ImportError:
     storage = None
 
-from ..logging import log
+from ..custom_logging import log
 from ..utils.gcp import is_running_on_gcp
 from ..auth.refresh import refresh_credentials, get_default_creds, get_default_email
 from io import BytesIO

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/gcs/metadata.py

@@ -3,7 +3,7 @@ try:
 except ImportError:
     storage = None
 
-from ..logging import log
+from ..custom_logging import log
 
 
 def get_object_metadata(bucket_name, object_name):

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/invoke/direct_vac_func.py

@@ -1,4 +1,4 @@
-from ..logging import log
+from ..custom_logging import log
 from ..agents import send_to_qa
 from ..qna.parsers import parse_output
 from ..streaming import generate_proxy_stream

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/invoke/invoke_vac_utils.py

@@ -3,7 +3,7 @@ import requests
 
 from pathlib import Path
 
-from ..logging import log
+from ..custom_logging import log
 
 def invoke_vac(service_url, data, vector_name=None, metadata=None, is_file=False):
     """

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/langfuse/callback.py

@@ -1,6 +1,6 @@
 
 from typing import Dict, Any
-from ..logging import log
+from ..custom_logging import log
 
 try:
     from langfuse.callback import CallbackHandler

{sunholo-0.79.0 → sunholo-0.79.2}/sunholo/langfuse/prompts.py

@@ -1,4 +1,4 @@
-from ..logging import log
+from ..custom_logging import log
 from ..utils import ConfigManager
 
 # Load the YAML file

sunholo-0.79.2/sunholo/llamaindex/__init__.py

@@ -0,0 +1 @@
+from .llamaindex_class import setup_llamaindex_subparser, LlamaIndexVertexCorpusManager