sunholo 0.80.5__tar.gz → 0.81.0__tar.gz

{sunholo-0.80.5 → sunholo-0.81.0}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.80.5
+Version: 0.81.0
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.80.5.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.81.0.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

sunholo-0.81.0/setup.cfg

@@ -0,0 +1,16 @@
+[metadata]
+description-file = README.md
+
+[mypy]
+files = sunholo, tests
+python_version = 3.9
+disallow_untyped_calls = True
+warn_return_any = True
+warn_unused_ignores = True
+strict_optional = True
+ignore_missing_imports = True
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

{sunholo-0.80.5 → sunholo-0.81.0}/setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 
 # Define your base version
-version = '0.80.5'
+version = '0.81.0'
 
 setup(
     name='sunholo',

{sunholo-0.80.5 → sunholo-0.81.0}/sunholo/chunker/process_chunker_data.py

@@ -15,6 +15,7 @@ from ..discovery_engine.chunker_handler import discovery_engine_chunker_check
 from .publish import process_docs_chunks_vector_name
 from .splitter import chunk_doc_to_docs
 from ..azure.blobs import is_azure_blob
+from ..utils import ConfigManager
 
 from ..custom_logging import log
 
@@ -29,13 +30,15 @@ def process_chunker_data(message_data, metadata, vector_name):
 
     log.debug(f"Found metadata in pubsub: {metadata=}")
 
+    config=ConfigManager(vector_name)
+
     # checks if only a llamaindex chunking/embedder, return early as no other processing needed
     llamacheck = llamaindex_chunker_check(message_data, metadata, vector_name)
     if llamacheck:
         return llamacheck
     
     # if only a discovery engine memory, return early as no other processing needed
-    discovery_check = discovery_engine_chunker_check(message_data, metadata, vector_name)
+    discovery_check = discovery_engine_chunker_check(message_data, metadata, config=config)
     if discovery_check:
         return discovery_check
 

{sunholo-0.80.5 → sunholo-0.81.0}/sunholo/components/retriever.py

@@ -170,4 +170,5 @@ def process_retrieval(retriever_list: list, config: ConfigManager):
         base_compressor=pipeline, base_retriever=lotr, 
         k=k_override)
     
+    log.info(f"Returning Langchain retrieval object: {retriever}")
     return retriever

{sunholo-0.80.5 → sunholo-0.81.0}/sunholo/database/__init__.py

@@ -4,3 +4,4 @@ from .database import setup_database
 from .database import return_sources_last24
 from .database import delete_row_from_source
 from .static_dbs import get_db_directory
+from .alloydb_client import AlloyDBClient

{sunholo-0.80.5 → sunholo-0.81.0}/sunholo/database/alloydb.py

@@ -14,6 +14,8 @@ from .alloydb_client import AlloyDBClient
 from ..custom_logging import log
 from ..utils.config import load_config_key
 
+from typing import List
+
 
 def create_alloydb_engine(vector_name):
 
@@ -231,8 +233,11 @@ async def load_alloydb_sql_async(sql, vector_name):
     
     return documents
 
-def and_or_ilike(sources, search_type="OR", operator="ILIKE"):
-    unique_sources = set(sources.split())
+def and_or_ilike(sources:List[str], search_type:str="OR", operator:str="ILIKE"):
+    if not isinstance(sources, list) or not all(isinstance(source, str) for source in sources):
+        raise TypeError("The `sources` argument must be a list of strings.")
+    
+    unique_sources = set(sources)
     # Choose the delimiter based on the search_type argument
     delimiter = ' AND ' if search_type.upper() == "AND" else ' OR '
 

{sunholo-0.80.5 → sunholo-0.81.0}/sunholo/database/alloydb_client.py

@@ -1,3 +1,4 @@
+import os
 try:
     import pg8000
     import sqlalchemy
@@ -9,6 +10,7 @@ except ImportError:
 
 from .database import get_vector_size
 from ..custom_logging import log
+from ..utils import ConfigManager
 
 class AlloyDBClient:
     """
@@ -35,11 +37,12 @@ class AlloyDBClient:
     """
 
     def __init__(self, 
-                 project_id: str, 
-                 region: str, 
-                 cluster_name:str, 
-                 instance_name:str, 
-                 user:str, 
+                 config:ConfigManager,
+                 project_id: str=None, 
+                 region: str=None, 
+                 cluster_name:str=None, 
+                 instance_name:str=None, 
+                 user:str=None, 
                  password=None, 
                  db="postgres"):
         """Initializes the AlloyDB client.
@@ -51,6 +54,23 @@ class AlloyDBClient:
          - password (str): The database user's password.
          - db_name (str): The name of the database.
         """
+        if config is None:
+            if project_id is None or region is None or cluster_name is None or instance_name is None:
+                raise ValueError("Must specify config or project_id, region, cluster_name, instance_name")
+        if config:
+            alloydb_config = config.vacConfig("alloydb_config")
+            if not alloydb_config:
+                raise ValueError("Must specify vac.alloydb_config")
+            project_id = alloydb_config["project_id"]
+            region = alloydb_config["region"]
+            cluster_name = alloydb_config["cluster"]
+            instance_name = alloydb_config["instance"]
+
+        ALLOYDB_DB = os.environ.get("ALLOYDB_DB")
+        if ALLOYDB_DB is None and alloydb_config.get("database") is None:
+            log.warning("Could not locate ALLOYDB_DB environment variable or 'alloydb_config.database'")
+        
+        self.database = alloydb_config.get("database") or ALLOYDB_DB,
         self.connector = Connector()
         self.inst_uri = self._build_instance_uri(project_id, region, cluster_name, instance_name)
         self.engine = self._create_engine(self.inst_uri, user, password, db)
@@ -100,6 +120,80 @@ class AlloyDBClient:
                 conn.close()
 
         return result
+    
+    async def execute_sql_async(self, sql_statement):
+        """Executes a given SQL statement asynchronously with error handling."""
+        sql_ = sqlalchemy.text(sql_statement)
+        result = None
+        async with self.engine.connect() as conn:
+            try:
+                log.info(f"Executing SQL statement asynchronously: {sql_}")
+                result = await conn.execute(sql_)
+            except DatabaseError as e:
+                if "already exists" in str(e):
+                    log.warning(f"Error ignored: {str(e)}. Assuming object already exists.")
+                else:
+                    raise
+            finally:
+                await conn.close()
+
+        return result
+
+    async def get_sources_from_docstore_async(self, sources, vector_name, search_type="OR", just_source_name=False):
+        """Fetches sources from the docstore asynchronously."""
+        if just_source_name:
+            query = self._list_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+        else:
+            query = self._get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+
+        if not query:
+            return []
+
+        documents = await self.execute_sql_async(query)
+        return documents
+
+    def _get_sources_from_docstore(self, sources, vector_name, search_type="OR"):
+        """Helper function to build the SQL query for fetching sources."""
+        if not sources:
+            log.warning("No sources found for alloydb fetch")
+            return ""
+
+        table_name = f"{vector_name}_docstore"
+
+        conditions = self._and_or_ilike(sources, search_type=search_type)
+
+        query = f"""
+            SELECT * 
+            FROM {table_name}
+            WHERE {conditions}
+            ORDER BY langchain_metadata->>'objectId' ASC
+            LIMIT 500;
+        """
+
+        return query
+
+    def _list_sources_from_docstore(self, sources, vector_name, search_type="OR"):
+        """Helper function to build the SQL query for listing sources."""
+        table_name = f"{vector_name}_docstore"
+
+        if sources:
+            conditions = self._and_or_ilike(sources, search_type=search_type)
+            query = f"""
+                SELECT DISTINCT langchain_metadata->>'objectId' AS objectId
+                FROM {table_name}
+                WHERE {conditions}
+                ORDER BY langchain_metadata->>'objectId' ASC
+                LIMIT 500;
+            """
+        else:
+            query = f"""
+                SELECT DISTINCT langchain_metadata->>'objectId' AS objectId
+                FROM {table_name}
+                ORDER BY langchain_metadata->>'objectId' ASC
+                LIMIT 500;
+            """
+
+        return query
 
     @staticmethod
     def _and_or_ilike(sources, search_type="OR", operator="ILIKE"):

{sunholo-0.80.5 → sunholo-0.81.0}/sunholo/discovery_engine/__init__.py

@@ -1 +1,2 @@
 from .discovery_engine_client import DiscoveryEngineClient
+from .get_ai_search_chunks import get_all_chunks

{sunholo-0.80.5 → sunholo-0.81.0}/sunholo/discovery_engine/chunker_handler.py

@@ -1,5 +1,5 @@
 from ..custom_logging import log
-from ..utils import load_config_key, ConfigManager
+from ..utils import ConfigManager
 from ..utils.gcp_project import get_gcp_project
 from ..components import load_memories
 
@@ -7,7 +7,7 @@ from .discovery_engine_client import DiscoveryEngineClient
 from .create_new import create_new_discovery_engine
     
 
-def do_discovery_engine(message_data, metadata, vector_name):
+def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=None):
     """
 
     Example:
@@ -15,13 +15,13 @@ def do_discovery_engine(message_data, metadata, vector_name):
     message_data = "gs://bucket_name/path_to_file.txt"
     metadata = {"user": "admin"}
     vector_name = "example_vector"
-    response = do_discovery_engine(message_data, metadata, vector_name)
+    response = do_discovery_engine(message_data, metadata, config=config)
     print(response)
     # Imported file to corpus: {'status': 'success'}
     ```
     """
 
-    memories = load_memories(vector_name)
+    memories = load_memories(config=config)
     tools = []
 
     if not memories:
@@ -38,7 +38,7 @@ def do_discovery_engine(message_data, metadata, vector_name):
                     continue
                 #location = gcp_config.get('location') 
                 corpus = DiscoveryEngineClient(
-                    data_store_id=vector_name, 
+                    data_store_id=config.vector_name, 
                     project_id=get_gcp_project(),
                     # location needs to be 'eu' or 'us' which doesn't work with other configurations
                     #location=location or global_location
@@ -65,14 +65,14 @@ def do_discovery_engine(message_data, metadata, vector_name):
                 log.error(f"Error importing {message_data} - {corp=} - {str(err)}")
 
                 if str(err).startswith("404"):
-                    log.info(f"Attempting to create a new DiscoveryEngine corpus: {vector_name}")
+                    log.info(f"Attempting to create a new DiscoveryEngine corpus: {config.vector_name}")
                     try:
-                        new_corp = create_new_discovery_engine(vector_name)
+                        new_corp = create_new_discovery_engine(config)
                     except Exception as err:
-                        log.error(f"Failed to create new DiscoveryEngine {vector_name} - {str(err)}")
+                        log.error(f"Failed to create new DiscoveryEngine {config.vector_name} - {str(err)}")
                         continue
                     if new_corp:
-                        log.info(f"Found new DiscoveryEngine {vector_name=} - {new_corp=}")
+                        log.info(f"Found new DiscoveryEngine {config.vector_name=} - {new_corp=}")
                         response = corp.import_documents(
                             gcs_uri=message_data
                         )
@@ -126,7 +126,7 @@ def discovery_engine_chunker_check(message_data, metadata, vector_name:str=None,
     total_memories = len(check_write_memories(config))
     llama = None
     if check_discovery_engine_in_memory(config):
-        llama = do_discovery_engine(message_data, metadata, vector_name)
+        llama = do_discovery_engine(message_data, metadata, config=config)
         log.info(f"Processed discovery engine: {llama}")
 
     # If discovery engine is the only entry, return

{sunholo-0.80.5 → sunholo-0.81.0}/sunholo/discovery_engine/create_new.py

@@ -1,10 +1,10 @@
 from .discovery_engine_client import DiscoveryEngineClient
-from ..utils.config import load_config_key
+from ..utils import ConfigManager
 from ..utils.gcp_project import get_gcp_project
 
-def create_new_discovery_engine(vector_name):
+def create_new_discovery_engine(config:ConfigManager):
 
-    chunker_config = load_config_key("chunker", vector_name=vector_name, kind="vacConfig")
+    chunker_config = config.vacConfig("chunker")
 
     chunk_size = 500
     if chunker_config:
@@ -18,7 +18,7 @@ def create_new_discovery_engine(vector_name):
     #location = gcp_config.get('location')
 
     de = DiscoveryEngineClient(
-                    data_store_id=vector_name, 
+                    data_store_id=config.vector_name, 
                     project_id=project_id,
                     # location needs to be 'eu' or 'us' which doesn't work with other configurations
                     #location=location

sunholo-0.81.0/sunholo/discovery_engine/get_ai_search_chunks.py

@@ -0,0 +1,53 @@
+from ..utils import ConfigManager
+from ..utils.gcp_project  import get_gcp_project
+from ..custom_logging import log
+from .discovery_engine_client import DiscoveryEngineClient
+from ..components import load_memories
+
+def get_all_chunks(question:str, config:ConfigManager):
+    """
+    Look through a config memory key and find all Vertex AI Search retrievers, call them and return a joined string of chunks
+
+        args: question - question to search similarity for
+        config: A ConfigManager object
+
+        returns: a big string of chunks
+    """
+    memories = load_memories(config=config)
+    chunks = []
+
+    if not memories:
+        return None
+
+    vector_name = config.vector_name
+    for memory in memories:
+        for key, value in memory.items():  # Now iterate over the dictionary
+            log.info(f"Found memory {key}")
+            vectorstore = value.get('vectorstore')
+            if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
+                if value.get('read_only'):
+                    new_vector_name = value.get('vector_name')
+                    if not new_vector_name:
+                        log.warning("read_only specified but no new vector_name to read from")
+                    vector_name = new_vector_name
+                
+                num_chunks = value.get('num_chunks') or 3
+
+                chunk = get_chunks(question, vector_name, num_chunks)
+                if chunk:
+                    chunks.append(chunk)
+    if chunks:
+        return "\n".join(chunks)
+
+    log.warning(f"No chunks found for {vector_name}")
+    return None
+
+def get_chunks(question, vector_name, num_chunks):
+    de = DiscoveryEngineClient(vector_name, project_id=get_gcp_project(include_config=True))
+    try:
+        return de.get_chunks(question, num_previous_chunks=num_chunks, num_next_chunks=num_chunks)
+    except Exception as err:
+        log.error(f"No discovery engine chunks found: {str(err)}")
+    
+
+

{sunholo-0.80.5 → sunholo-0.81.0}/sunholo/utils/config_class.py

@@ -37,9 +37,10 @@ class ConfigManager:
         self.config_folder = os.getenv("VAC_CONFIG_FOLDER", os.getcwd())
         self.local_config_folder = local_config_folder
         self.configs_by_kind = self.load_all_configs()
+        self.validate = validate
 
         test_agent = self.vacConfig("agent")
-        if not test_agent and self.vector_name != "global" and validate:
+        if not test_agent and self.vector_name != "global" and self.validate:
             print(f"WARNING: No vacConfig.agent found for {self.vector_name} - are you in right folder? {local_config_folder=} {self.config_folder=}")
 
     def load_all_configs(self):
@@ -125,7 +126,6 @@ class ConfigManager:
             else:
                 config = yaml.safe_load(file)
         self.config_cache[filename] = (config, datetime.now())
-        log.debug(f"Loaded and cached {config_file}")
         if is_local:
             log.warning(f"Local configuration override for {filename} via {self.local_config_folder}")
         return config

{sunholo-0.80.5 → sunholo-0.81.0}/sunholo/utils/gcp_project.py

@@ -24,6 +24,7 @@ def get_gcp_project(include_config=False):
         gcp_config = load_config_key("gcp_config", "global", "vacConfig")
         if gcp_config:
             if gcp_config.get('project_id'):
+                logging.info("Using project_id from vacConfig.gcp_config.project_id")
                 return gcp_config.get('project_id')
 
     project_id = get_env_project_id()

{sunholo-0.80.5 → sunholo-0.81.0}/sunholo.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.80.5
+Version: 0.81.0
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.80.5.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.81.0.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.80.5 → sunholo-0.81.0}/sunholo.egg-info/SOURCES.txt

@@ -85,6 +85,7 @@ sunholo/discovery_engine/__init__.py
 sunholo/discovery_engine/chunker_handler.py
 sunholo/discovery_engine/create_new.py
 sunholo/discovery_engine/discovery_engine_client.py
+sunholo/discovery_engine/get_ai_search_chunks.py
 sunholo/embedder/__init__.py
 sunholo/embedder/embed_chunk.py
 sunholo/gcs/__init__.py

sunholo-0.80.5/setup.cfg

@@ -1,7 +0,0 @@
-[metadata]
-description-file = README.md
-
-[egg_info]
-tag_build = 
-tag_date = 0
-