sunholo 0.78.5__py3-none-any.whl → 0.79.1__py3-none-any.whl

sunholo/agents/chat_history.py

@@ -1,5 +1,5 @@
 import json
-from ..logging import log
+from ..custom_logging import log
 
 def extract_chat_history(chat_history=None):
     """

sunholo/agents/dispatch_to_qa.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 from ..utils import ConfigManager
 from ..auth import get_header
 import requests

sunholo/agents/fastapi/base.py

@@ -11,7 +11,7 @@ try:
 except ImportError:
     FastAPI = None
 
-from ...logging import log
+from ...custom_logging import log
 
 def create_fastapi_app(origins = ["*"],
                        origin_regex = r"https://(.*\.)?sunholo\.com"):

sunholo/agents/fastapi/qna_routes.py

@@ -15,7 +15,7 @@ from ...agents import extract_chat_history, handle_special_commands
 from ...qna.parsers import parse_output
 from ...streaming import start_streaming_chat_async
 from ...archive import archive_qa
-from ...logging import log
+from ...custom_logging import log
 
 class VACRequest(BaseModel):
     user_input: str

sunholo/agents/flask/qna_routes.py

@@ -21,7 +21,7 @@ from ...agents import extract_chat_history, handle_special_commands
 from ...qna.parsers import parse_output
 from ...streaming import start_streaming_chat
 from ...archive import archive_qa
-from ...logging import log
+from ...custom_logging import log
 from ...utils.config import load_config
 from ...utils.version import sunholo_version
 import os

sunholo/agents/flask/vac_routes.py

@@ -7,7 +7,7 @@ from ...agents import extract_chat_history, handle_special_commands
 from ...qna.parsers import parse_output
 from ...streaming import start_streaming_chat
 from ...archive import archive_qa
-from ...logging import log
+from ...custom_logging import log
 from ...utils.config import load_config
 from ...utils.version import sunholo_version
 import os

sunholo/agents/langserve.py

@@ -1,5 +1,5 @@
 import requests
-from ..logging import log
+from ..custom_logging import log
 from ..auth import get_header
 
 # Global cache for storing input schemas

sunholo/agents/pubsub.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 
 
 import base64

sunholo/agents/route.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 from ..utils import load_config, ConfigManager
 
 def read_cloud_run_url(agent, cloud_run_urls_file='config/cloud_run_urls.json'):

sunholo/agents/special_commands.py

@@ -26,7 +26,7 @@ from ..utils.parsers import contains_url, extract_urls
 from ..chunker.publish import publish_text
 from ..gcs.add_file import add_file_to_gcs
 from ..utils.config import load_config_key
-from ..logging import log
+from ..custom_logging import log
 
 # config file?
 command_descriptions = {

sunholo/agents/swagger.py

@@ -1,7 +1,7 @@
 import copy
 from ..utils.config import load_all_configs
 from .route import route_vac
-from ..logging import log
+from ..custom_logging import log
 from ruamel.yaml import YAML
 from io import StringIO
 # check it here:

sunholo/archive/archive.py

@@ -12,7 +12,7 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 from ..pubsub import PubSubManager
-from ..logging import log
+from ..custom_logging import log
 
 
 

sunholo/auth/refresh.py

@@ -6,7 +6,7 @@ from google.auth.transport import requests
 from ..utils.gcp import is_running_on_gcp
 
 
-from ..logging import log
+from ..custom_logging import log
 
 def get_default_email():
     if not refresh_credentials():

sunholo/auth/run.py

@@ -6,7 +6,7 @@ from ..utils.config import load_config
 from ..utils import ConfigManager
 from ..utils.gcp import is_running_on_cloudrun
 from ..utils.api_key import has_multivac_api_key, get_multivac_api_key
-from ..logging import log
+from ..custom_logging import log
 from ..agents.route import route_vac
 from .gcloud import get_local_gcloud_token
 

sunholo/azure/auth.py

@@ -5,7 +5,7 @@ except ImportError:
     DefaultAzureCredential = None
     ClientSecretCredential = None
 
-from ..logging import log
+from ..custom_logging import log
 
 def azure_auth():
     """

sunholo/azure/blobs.py

@@ -1,5 +1,5 @@
 import re
-from ..logging import log
+from ..custom_logging import log
 
 
 def is_azure_blob(message_data):

sunholo/azure/event_grid.py

@@ -1,5 +1,5 @@
 # process_azure_blob_event.py
-from ..logging import log
+from ..custom_logging import log
 
 def process_azure_blob_event(events: list) -> tuple:
     """

sunholo/bots/discord.py

@@ -13,7 +13,7 @@
 #   limitations under the License.
 import json, os
 import requests
-from ..logging import log
+from ..custom_logging import log
 
 
 

sunholo/chunker/azure.py

@@ -14,7 +14,7 @@
 import json
 from datetime import datetime, timezone
 
-from ..logging import log
+from ..custom_logging import log
 from ..azure import process_azure_blob_event
 
 from ..invoke import invoke_vac

sunholo/chunker/doc_handling.py

@@ -1,5 +1,5 @@
 from ..utils import load_config_key
-from ..logging import log
+from ..custom_logging import log
 from ..database.alloydb import add_document_if_not_exists
 from ..database.uuid import generate_uuid_from_object_id
 from ..components.llm import llm_str_to_llm
@@ -175,6 +175,10 @@ Be careful not to add any speculation or any details that are not covered in the
                 bucket_name = os.getenv("DOC_BUCKET")
                 if not bucket_name:
                     raise ValueError("No DOC_BUCKET configured for summary")
+                
+                if bucket_name.startswith("gs://"):
+                    bucket_name = bucket_name[len("gs://"):]
+
                 with tempfile.NamedTemporaryFile(mode='w+', delete=False) as temp_file:
                     temp_file.write(summary)
                     temp_file.flush()

sunholo/chunker/encode_metadata.py

@@ -3,7 +3,7 @@ import base64
 import json
 from datetime import datetime, timezone
 
-from ..logging import log
+from ..custom_logging import log
 
 def create_metadata(vac, metadata):
     now_utc = datetime.now(timezone.utc)

sunholo/chunker/images.py

@@ -3,7 +3,7 @@ import datetime
 import tempfile
 import os
 from ..gcs.add_file import add_file_to_gcs, get_image_file_name
-from ..logging import log
+from ..custom_logging import log
 from ..utils.gcp import is_running_on_gcp
 
 

sunholo/chunker/loaders.py

@@ -18,7 +18,7 @@ from langchain_community.document_loaders import UnstructuredURLLoader
 from langchain_community.document_loaders import GitLoader
 from langchain_community.document_loaders import GoogleDriveLoader
 
-from ..logging import log
+from ..custom_logging import log
 from .pdfs import read_pdf_file
 from ..utils.config import load_config
 

sunholo/chunker/message_data.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 
 import pathlib
 import tempfile

sunholo/chunker/pdfs.py

@@ -13,7 +13,7 @@
 #   limitations under the License.
 import os
 import pathlib
-from ..logging import log
+from ..custom_logging import log
 
 def split_pdf_to_pages(pdf_path, temp_dir):
 

sunholo/chunker/process_chunker_data.py

@@ -16,7 +16,7 @@ from .publish import process_docs_chunks_vector_name
 from .splitter import chunk_doc_to_docs
 from ..azure.blobs import is_azure_blob
 
-from ..logging import log
+from ..custom_logging import log
 
 def process_chunker_data(message_data, metadata, vector_name):
 

sunholo/chunker/publish.py

@@ -1,4 +1,4 @@
-from ..logging import log
+from ..custom_logging import log
 from ..pubsub import PubSubManager
 from ..utils.parsers import contains_url, extract_urls
 from ..utils.gcp_project import get_gcp_project

sunholo/chunker/pubsub.py

@@ -12,7 +12,7 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-from ..logging import log
+from ..custom_logging import log
 from ..pubsub import process_pubsub_message
 from .process_chunker_data import process_chunker_data
 

sunholo/chunker/splitter.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 from ..utils.parsers import remove_whitespace
 from langchain.schema import Document
 import langchain.text_splitter as text_splitter

sunholo/cli/chat_vac.py

@@ -3,7 +3,7 @@ from ..streaming import generate_proxy_stream, can_agent_stream
 from ..utils.user_ids import generate_user_id
 from ..utils import ConfigManager
 from ..utils.api_key import has_multivac_api_key
-from ..logging import log
+from ..custom_logging import log
 from ..qna.parsers import parse_output
 from ..gcs.add_file import add_file_to_gcs
 from .run_proxy import clean_proxy_list, start_proxy, stop_proxy

sunholo/cli/cli.py

@@ -10,10 +10,11 @@ from .chat_vac import setup_vac_subparser
 from .embedder import setup_embedder_subparser
 from .swagger import setup_swagger_subparser
 from .vertex import setup_vertex_subparser
+from ..llamaindex import setup_llamaindex_subparser
 
 from ..utils import ConfigManager
 
-from ..logging import log
+from ..custom_logging import log
 
 from .sun_rich import console
 import sys
@@ -87,6 +88,8 @@ def main(args=None):
     setup_swagger_subparser(subparsers)
     # vertex
     setup_vertex_subparser(subparsers)
+    # llamaindex
+    setup_llamaindex_subparser(subparsers)
 
     #TODO: add database setup commands: alloydb and supabase
 

sunholo/components/llm.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 from ..utils import load_config_key, ConfigManager
 
 import os

sunholo/components/retriever.py

@@ -11,7 +11,7 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
-from ..logging import log
+from ..custom_logging import log
 from .vectorstore import pick_vectorstore
 from ..utils import load_config_key, ConfigManager
 from .llm import get_embeddings
@@ -41,9 +41,14 @@ def load_memories(vector_name:str=None, config:ConfigManager=None):
 
     return memories
 
-def pick_retriever(vector_name, embeddings=None):
+def pick_retriever(vector_name:str=None, config:ConfigManager=None, embeddings=None):
 
-    memories = load_memories(vector_name)
+    if config is None:
+        if vector_name is None:
+            raise ValueError("vector_name and config were none")
+        config = ConfigManager(vector_name)
+
+    memories = load_memories(config=config)
 
     retriever_list = []
     for memory in memories:  # Iterate over the list
@@ -58,7 +63,7 @@ def pick_retriever(vector_name, embeddings=None):
                     log.info(f"Skipped from_metadata_id for {vectorstore}")
                     continue
 
-                embeddings = embeddings or get_embeddings(vector_name)
+                embeddings = embeddings or get_embeddings(config=config)
                 read_only = value.get('read_only')
                 try:
                     vectorstore = pick_vectorstore(vectorstore, 
@@ -93,15 +98,15 @@ def pick_retriever(vector_name, embeddings=None):
         log.info(f"No retrievers were created for {memories}")
         return None
     
-    retriever = process_retrieval(retriever_list, vector_name)
+    retriever = process_retrieval(retriever_list, config=config)
 
     return retriever
 
-def metadata_retriever(metadata: dict, key: str, vector_name:str, embeddings=None):
+def metadata_retriever(metadata: dict, key: str, config:ConfigManager, embeddings=None):
     """
     Decides which vector_name to retrieve from metadata passed
     """
-    memories = load_memories(vector_name)
+    memories = load_memories(config=config)
 
     retriever_list = []
     for memory in memories:  # Iterate over the list
@@ -118,7 +123,7 @@ def metadata_retriever(metadata: dict, key: str, vector_name:str, embeddings=Non
                         raise ValueError(f"Missing {key} in {metadata}")
                     the_id = metadata[key]
                     read_only = value.get('read_only')
-                    embeddings = embeddings or get_embeddings(vector_name)
+                    embeddings = embeddings or get_embeddings(config=config)
                     vectorstore = pick_vectorstore(vectorstore, 
                                                    vector_name=the_id, 
                                                    embeddings=embeddings, 
@@ -133,17 +138,18 @@ def metadata_retriever(metadata: dict, key: str, vector_name:str, embeddings=Non
         log.info(f"No retrievers were created for {memories}")
         return None
     
-    retriever = process_retrieval(retriever_list, vector_name)
+    retriever = process_retrieval(retriever_list, config=config)
 
     return retriever
     
 
 
-def process_retrieval(retriever_list: list, vector_name: str):
-    k_override = load_config_key("memory_k", vector_name, kind="vacConfig")
+def process_retrieval(retriever_list: list, config: ConfigManager):
+    k_override = config.vacConfig('memory_k')
     lotr = MergerRetriever(retrievers=retriever_list)
 
-    filter_embeddings = get_embeddings(vector_name)
+    filter_embeddings = get_embeddings(config=config)
+
     filter = EmbeddingsRedundantFilter(embeddings=filter_embeddings)
     pipeline = DocumentCompressorPipeline(transformers=[filter])
     retriever = ContextualCompressionRetriever(

sunholo/components/vectorstore.py

@@ -12,10 +12,21 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 import os
-from ..logging import log
+from ..custom_logging import log
+from ..utils import ConfigManager
 
-def pick_vectorstore(vs_str: str, vector_name: str, embeddings, read_only=None):
+def pick_vectorstore(vs_str: str, vector_name: str=None, embeddings=None, config:ConfigManager=None, read_only=None):
     log.debug('Picking vectorstore')
+
+    if not embeddings:
+        raise ValueError("Requires embeddings")
+    
+    if config is None:
+        if vector_name is None:
+            raise ValueError("vector_name and config were none")
+        config = ConfigManager(vector_name)
+        
+    vector_name = config.vector_name
         
     if vs_str == 'supabase':
         from supabase import Client, create_client

sunholo/database/alloydb.py

@@ -11,7 +11,7 @@ except ImportError:
 from .database import get_vector_size
 from .alloydb_client import AlloyDBClient
 
-from ..logging import log
+from ..custom_logging import log
 from ..utils.config import load_config_key
 
 
@@ -232,7 +232,7 @@ async def load_alloydb_sql_async(sql, vector_name):
     return documents
 
 def and_or_ilike(sources, search_type="OR", operator="ILIKE"):
-    unique_sources = set(sources)
+    unique_sources = set(sources.split())
     # Choose the delimiter based on the search_type argument
     delimiter = ' AND ' if search_type.upper() == "AND" else ' OR '
 
@@ -240,14 +240,14 @@ def and_or_ilike(sources, search_type="OR", operator="ILIKE"):
     conditions = delimiter.join(f"TRIM(source) {operator} '%{source}%'" for source in unique_sources)
     if not conditions:
         log.warning("Alloydb doc query found no like_patterns")
-        return []
+        return ""
     
     return conditions
 
 def _get_sources_from_docstore(sources, vector_name, search_type="OR"):
     if not sources:
         log.warning("No sources found for alloydb fetch")
-        return []
+        return ""
 
     table_name = f"{vector_name}_docstore"
 
@@ -263,10 +263,37 @@ def _get_sources_from_docstore(sources, vector_name, search_type="OR"):
 
     return query
 
+def _list_sources_from_docstore(sources, vector_name, search_type="OR"):
+    table_name = f"{vector_name}_docstore"
+    
+
+    if sources:
+        conditions = and_or_ilike(sources, search_type=search_type)
+        query = f"""
+            SELECT DISTINCT langchain_metadata->>'objectId' AS objectId
+            FROM {table_name}
+            WHERE {conditions}
+            ORDER BY langchain_metadata->>'objectId' ASC
+            LIMIT 500;
+        """
+    else:
+        query = f"""
+            SELECT DISTINCT langchain_metadata->>'objectId' AS objectId
+            FROM {table_name}
+            ORDER BY langchain_metadata->>'objectId' ASC
+            LIMIT 500;
+        """
+
+    return query
 
-async def get_sources_from_docstore_async(sources, vector_name, search_type="OR"):
+
+async def get_sources_from_docstore_async(sources, vector_name, search_type="OR", just_source_name=False):
     
-    query = _get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+    if just_source_name:
+        query = _list_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+    else:
+        query = _get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+
     if not query:
         return []
     
@@ -274,9 +301,13 @@ async def get_sources_from_docstore_async(sources, vector_name, search_type="OR"
     
     return documents
 
-def get_sources_from_docstore(sources, vector_name, search_type="OR"):
+def get_sources_from_docstore(sources, vector_name, search_type="OR", just_source_name=False):
     
-    query = _get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+    if just_source_name:
+        query = _list_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+    else:
+        query = _get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+
     if not query:
         return []
     
@@ -303,3 +334,5 @@ def delete_sources_from_alloydb(sources, vector_name):
         DELETE FROM {vector_name}_vectorstore_{vector_length}
         WHERE {conditions}
     """
+
+    return query

sunholo/database/alloydb_client.py

@@ -8,7 +8,7 @@ except ImportError:
     pass
 
 from .database import get_vector_size
-from ..logging import log
+from ..custom_logging import log
 
 class AlloyDBClient:
     """

sunholo/database/database.py

@@ -17,7 +17,7 @@ import time
 import math
 
 from ..utils.config import get_module_filepath
-from ..logging import log
+from ..custom_logging import log
 from ..utils.config import load_config_key
 
 

sunholo/database/lancedb.py

@@ -1,4 +1,4 @@
-from ..logging import log
+from ..custom_logging import log
 
 def create_lancedb_index(bucket, vector_name, num_partitions=256, num_sub_vectors=96, text_key="text"):
     try:

sunholo/database/static_dbs.py

@@ -1,5 +1,5 @@
 import os
-from ..logging import log_folder_location
+from ..custom_logging import log_folder_location
 
 def get_db_directory(db_dir='db'):
     current_script_directory = os.path.dirname(os.path.abspath(__file__))

sunholo/discovery_engine/chunker_handler.py

@@ -1,4 +1,4 @@
-from ..logging import log
+from ..custom_logging import log
 from ..utils.config import load_config_key
 from ..utils.gcp_project import get_gcp_project
 from ..components import load_memories

sunholo/discovery_engine/discovery_engine_client.py

@@ -10,7 +10,7 @@ except ImportError:
     Chunk = None
     SearchResponse = None
 
-from ..logging import log
+from ..custom_logging import log
 from typing import Optional, List
 
 class DiscoveryEngineClient:

sunholo/embedder/embed_chunk.py

@@ -20,7 +20,7 @@ import uuid
 from langchain.schema import Document
 
 from ..components import get_embeddings, pick_vectorstore, load_memories, pick_embedding
-from ..logging import log
+from ..custom_logging import log
 from ..database.uuid import generate_uuid_from_object_id
 from ..utils import ConfigManager
 

sunholo/gcs/add_file.py

@@ -21,7 +21,7 @@ try:
 except ImportError:
     storage = None
 
-from ..logging import log
+from ..custom_logging import log
 from ..utils import load_config_key, ConfigManager
 
 

sunholo/gcs/download_folder.py

@@ -5,7 +5,7 @@ try:
 except ImportError:
     storage = None
 
-from ..logging import log
+from ..custom_logging import log
 
 def download_files_from_gcs(bucket_name: str, source_folder: str, destination_folder: str=None):
     """

sunholo/gcs/download_url.py

@@ -10,7 +10,7 @@ try:
 except ImportError:
     storage = None
 
-from ..logging import log
+from ..custom_logging import log
 from ..utils.gcp import is_running_on_gcp
 from ..auth.refresh import refresh_credentials, get_default_creds, get_default_email
 from io import BytesIO

sunholo/gcs/metadata.py

@@ -3,7 +3,7 @@ try:
 except ImportError:
     storage = None
 
-from ..logging import log
+from ..custom_logging import log
 
 
 def get_object_metadata(bucket_name, object_name):

sunholo/invoke/direct_vac_func.py

@@ -1,4 +1,4 @@
-from ..logging import log
+from ..custom_logging import log
 from ..agents import send_to_qa
 from ..qna.parsers import parse_output
 from ..streaming import generate_proxy_stream