sunholo 0.78.4__tar.gz → 0.79.0__tar.gz

{sunholo-0.78.4 → sunholo-0.79.0}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.78.4
+Version: 0.79.0
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.78.4.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.0.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.78.4 → sunholo-0.79.0}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 # Define your base version
-version = '0.78.4'
+version = '0.79.0'
 
 setup(
     name='sunholo',

{sunholo-0.78.4 → sunholo-0.79.0}/sunholo/chunker/doc_handling.py

@@ -175,6 +175,10 @@ Be careful not to add any speculation or any details that are not covered in the
                 bucket_name = os.getenv("DOC_BUCKET")
                 if not bucket_name:
                     raise ValueError("No DOC_BUCKET configured for summary")
+                
+                if bucket_name.startswith("gs://"):
+                    bucket_name = bucket_name[len("gs://"):]
+                    
                 with tempfile.NamedTemporaryFile(mode='w+', delete=False) as temp_file:
                     temp_file.write(summary)
                     temp_file.flush()

{sunholo-0.78.4 → sunholo-0.79.0}/sunholo/components/llm.py

@@ -16,15 +16,22 @@ from ..utils import load_config_key, ConfigManager
 
 import os
 
-def pick_llm(vector_name):
+def pick_llm(vector_name:str=None, config:ConfigManager=None):
+
+    if config is None:
+        if vector_name is None:
+            raise ValueError("config and vector_name was None")
+        config = ConfigManager(vector_name)
+
     log.debug('Picking llm')
     
-    llm_str = load_config_key("llm", vector_name, kind="vacConfig")
+    llm_str = config.vacConfig("llm")
     
     if llm_str == 'openai':
-        llm_chat = get_llm_chat(vector_name)
-        llm = get_llm_chat(vector_name, model="gpt-3.5-turbo-16k") # TODO: fix it needs llm_chat and not llm
-        embeddings = get_embeddings(vector_name)
+        llm_chat = get_llm_chat(config=config)
+        llm = get_llm_chat(model="gpt-3.5-turbo-16k", config=config) # TODO: fix it needs llm_chat and not llm
+        embeddings = get_embeddings(config=config)
+
         log.debug("Chose OpenAI")
     elif llm_str == 'vertex':
         llm = get_llm_chat(vector_name) # TODO: fix it needs llm_chat and not llm
@@ -62,7 +69,9 @@ def llm_str_to_llm(llm_str, model=None, vector_name=None, config=None):
     if llm_str is None:
         raise NotImplementedError("llm_str was None")
     
-    if vector_name:
+    if config is None:
+        if vector_name is None:
+            raise ValueError("vector_name and config was None")
         config = ConfigManager(vector_name)
 
     if llm_str == 'openai':
@@ -125,10 +134,16 @@ def get_llm(vector_name=None, model=None, config=None):
     log.debug(f"Chose LLM: {llm_str}")
     return llm_str_to_llm(llm_str, model=model, config=config)
 
-def get_llm_chat(vector_name, model=None):
-    llm_str = load_config_key("llm", vector_name, kind="vacConfig")
+def get_llm_chat(vector_name:str=None, model=None, config:ConfigManager=None):
+
+    if config is None:
+        if vector_name is None:
+            raise ValueError("config and vector_name was None")
+        config = ConfigManager(vector_name)
+
+    llm_str = config.vacConfig("llm")
     if not model:
-        model = load_config_key("model", vector_name, kind="vacConfig")
+        model = config.vacConfig("model")
 
     log.debug(f"Chose LLM: {llm_str}")
     # Configure LLMs based on llm_str
@@ -167,7 +182,7 @@ def get_llm_chat(vector_name, model=None):
         return ChatAnthropic(model_name = model, temperature=0)
     elif llm_str == 'azure':
         from langchain_openai import AzureChatOpenAI
-        azure_config = load_config_key("azure", vector_name, kind="vacConfig")
+        azure_config = config.vacConfig("azure")
         if not azure_config:
             raise ValueError("Need to configure azure.config if llm='azure'")
 
@@ -209,22 +224,37 @@ def get_llm_chat(vector_name, model=None):
     if llm_str is None:
         raise NotImplementedError(f'No llm implemented for {llm_str}')
 
-def get_embeddings(vector_name):
+def get_embeddings(vector_name=None, config:ConfigManager=None):
+
+    if not config:
+        if not vector_name:
+            raise ValueError(f"config and vector_name was None: {vector_name}")
+        config = ConfigManager(vector_name)
+    
 
     llm_str = None
-    embed_dict = load_config_key("embedder", vector_name, kind="vacConfig")
+    embed_dict = config.vacConfig("embedder")
 
     if embed_dict:
         llm_str = embed_dict.get('llm')
 
     if llm_str is None:
-        llm_str = load_config_key("llm", vector_name, kind="vacConfig")
+        llm_str = config.vacConfig("llm")
+    
+    if llm_str is None:
+        raise ValueError(f"llm_str was None: {llm_str}")
 
-    return pick_embedding(llm_str, vector_name=vector_name)
+    return pick_embedding(llm_str, config=config)
 
 
 #TODO: specify model
-def pick_embedding(llm_str: str, vector_name: str=None):
+def pick_embedding(llm_str: str, vector_name: str=None, config: ConfigManager=None):
+
+    if not config:
+        if not vector_name:
+            raise ValueError(f"config and vector_name was None {vector_name}")
+        config = ConfigManager(vector_name)
+
     # get embedding directly from llm_str
     # Configure embeddings based on llm_str
     if llm_str == 'openai':
@@ -244,7 +274,7 @@ def pick_embedding(llm_str: str, vector_name: str=None):
     elif llm_str == 'azure':
         from langchain_openai import AzureOpenAIEmbeddings
         
-        azure_config = load_config_key("azure", vector_name, kind="vacConfig")
+        azure_config = config.vacConfig("azure")
         if not azure_config:
             raise ValueError("Need to configure azure.config if llm='azure'")
 

{sunholo-0.78.4 → sunholo-0.79.0}/sunholo/components/retriever.py

@@ -26,8 +26,14 @@ from langchain.retrievers import ContextualCompressionRetriever
 
 
 
-def load_memories(vector_name):
-    memories = ConfigManager(vector_name).vacConfig("memory")
+def load_memories(vector_name:str=None, config:ConfigManager=None):
+    if config is None:
+        if vector_name is None:
+            raise ValueError("vector_name and config were none")
+        config = ConfigManager(vector_name)
+
+    memories = config.vacConfig("memory")
+    
     log.info(f"Found memory settings for {vector_name}: {memories}")
     if not memories or len(memories) == 0:
         log.info(f"No memory settings found for {vector_name}")

{sunholo-0.78.4 → sunholo-0.79.0}/sunholo/database/alloydb.py

@@ -232,7 +232,7 @@ async def load_alloydb_sql_async(sql, vector_name):
     return documents
 
 def and_or_ilike(sources, search_type="OR", operator="ILIKE"):
-    unique_sources = set(sources)
+    unique_sources = set(sources.split())
     # Choose the delimiter based on the search_type argument
     delimiter = ' AND ' if search_type.upper() == "AND" else ' OR '
 
@@ -240,14 +240,14 @@ def and_or_ilike(sources, search_type="OR", operator="ILIKE"):
     conditions = delimiter.join(f"TRIM(source) {operator} '%{source}%'" for source in unique_sources)
     if not conditions:
         log.warning("Alloydb doc query found no like_patterns")
-        return []
+        return ""
     
     return conditions
 
 def _get_sources_from_docstore(sources, vector_name, search_type="OR"):
     if not sources:
         log.warning("No sources found for alloydb fetch")
-        return []
+        return ""
 
     table_name = f"{vector_name}_docstore"
 
@@ -263,10 +263,37 @@ def _get_sources_from_docstore(sources, vector_name, search_type="OR"):
 
     return query
 
+def _list_sources_from_docstore(sources, vector_name, search_type="OR"):
+    table_name = f"{vector_name}_docstore"
+    
+
+    if sources:
+        conditions = and_or_ilike(sources, search_type=search_type)
+        query = f"""
+            SELECT DISTINCT langchain_metadata->>'objectId' AS objectId
+            FROM {table_name}
+            WHERE {conditions}
+            ORDER BY langchain_metadata->>'objectId' ASC
+            LIMIT 500;
+        """
+    else:
+        query = f"""
+            SELECT DISTINCT langchain_metadata->>'objectId' AS objectId
+            FROM {table_name}
+            ORDER BY langchain_metadata->>'objectId' ASC
+            LIMIT 500;
+        """
+
+    return query
 
-async def get_sources_from_docstore_async(sources, vector_name, search_type="OR"):
+
+async def get_sources_from_docstore_async(sources, vector_name, search_type="OR", just_source_name=False):
     
-    query = _get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+    if just_source_name:
+        query = _list_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+    else:
+        query = _get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+
     if not query:
         return []
     
@@ -274,9 +301,13 @@ async def get_sources_from_docstore_async(sources, vector_name, search_type="OR"
     
     return documents
 
-def get_sources_from_docstore(sources, vector_name, search_type="OR"):
+def get_sources_from_docstore(sources, vector_name, search_type="OR", just_source_name=False):
     
-    query = _get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+    if just_source_name:
+        query = _list_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+    else:
+        query = _get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+
     if not query:
         return []
     
@@ -303,3 +334,5 @@ def delete_sources_from_alloydb(sources, vector_name):
         DELETE FROM {vector_name}_vectorstore_{vector_length}
         WHERE {conditions}
     """
+
+    return query

{sunholo-0.78.4 → sunholo-0.79.0}/sunholo/embedder/embed_chunk.py

@@ -22,6 +22,7 @@ from langchain.schema import Document
 from ..components import get_embeddings, pick_vectorstore, load_memories, pick_embedding
 from ..logging import log
 from ..database.uuid import generate_uuid_from_object_id
+from ..utils import ConfigManager
 
 def embed_pubsub_chunk(data: dict):
     """Triggered from a message on a Cloud Pub/Sub topic "embed_chunk" topic
@@ -63,6 +64,9 @@ def embed_pubsub_chunk(data: dict):
         log.error(msg)
         return msg
     
+    config = ConfigManager(vector_name)
+    log.info(f"{config=}")
+    
     log.info(f"Embedding: {vector_name} page_content: {page_content[:30]}...[{len(page_content)}] - {metadata}")
 
     if 'eventTime' not in metadata:
@@ -102,9 +106,9 @@ def embed_pubsub_chunk(data: dict):
     doc = Document(page_content=page_content, metadata=metadata)
 
     # init embedding and vector store
-    embeddings = get_embeddings(vector_name)
+    embeddings = get_embeddings(config=config)
 
-    memories = load_memories(vector_name)
+    memories = load_memories(config=config)
     vectorstore_list = []
     for memory in memories:  # Iterate over the list
         for key, value in memory.items(): 
@@ -114,7 +118,7 @@ def embed_pubsub_chunk(data: dict):
                 # check if vectorstore specific embedding is available
                 embed_llm = value.get('llm')
                 if embed_llm:
-                    embeddings = pick_embedding(embed_llm)
+                    embeddings = pick_embedding(embed_llm, config=config)
                 # check if read only
                 read_only = value.get('read_only')
                 if read_only:

{sunholo-0.78.4 → sunholo-0.79.0}/sunholo/utils/config_class.py

@@ -37,6 +37,10 @@ class ConfigManager:
         self.local_config_folder = local_config_folder
         self.configs_by_kind = self.load_all_configs()
 
+        test_agent = self.vacConfig("agent")
+        if not test_agent:
+            print(f"WARNING: No vacConfig.agent found for {self.vector_name} - are you in right folder? {local_config_folder=} {self.config_folder=}")
+
     def load_all_configs(self):
         """
         Load all configuration files from the specified directories into a dictionary.
@@ -121,7 +125,7 @@ class ConfigManager:
         self.config_cache[filename] = (config, datetime.now())
         log.debug(f"Loaded and cached {config_file}")
         if is_local:
-            log.warning(f"Local configuration override for {filename}")
+            log.warning(f"Local configuration override for {filename} via {self.local_config_folder}")
         return config
 
     def _check_and_reload_configs(self):

{sunholo-0.78.4 → sunholo-0.79.0}/sunholo.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.78.4
+Version: 0.79.0
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.78.4.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.0.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0