sunholo 0.80.6__py3-none-any.whl → 0.81.0__py3-none-any.whl

sunholo/components/retriever.py

@@ -170,4 +170,5 @@ def process_retrieval(retriever_list: list, config: ConfigManager):
         base_compressor=pipeline, base_retriever=lotr, 
         k=k_override)
     
+    log.info(f"Returning Langchain retrieval object: {retriever}")
     return retriever

sunholo/database/__init__.py

@@ -4,3 +4,4 @@ from .database import setup_database
 from .database import return_sources_last24
 from .database import delete_row_from_source
 from .static_dbs import get_db_directory
+from .alloydb_client import AlloyDBClient

sunholo/database/alloydb.py

@@ -14,6 +14,8 @@ from .alloydb_client import AlloyDBClient
 from ..custom_logging import log
 from ..utils.config import load_config_key
 
+from typing import List
+
 
 def create_alloydb_engine(vector_name):
 
@@ -231,8 +233,11 @@ async def load_alloydb_sql_async(sql, vector_name):
     
     return documents
 
-def and_or_ilike(sources, search_type="OR", operator="ILIKE"):
-    unique_sources = set(sources.split())
+def and_or_ilike(sources:List[str], search_type:str="OR", operator:str="ILIKE"):
+    if not isinstance(sources, list) or not all(isinstance(source, str) for source in sources):
+        raise TypeError("The `sources` argument must be a list of strings.")
+    
+    unique_sources = set(sources)
     # Choose the delimiter based on the search_type argument
     delimiter = ' AND ' if search_type.upper() == "AND" else ' OR '
 

sunholo/database/alloydb_client.py

@@ -1,3 +1,4 @@
+import os
 try:
     import pg8000
     import sqlalchemy
@@ -9,6 +10,7 @@ except ImportError:
 
 from .database import get_vector_size
 from ..custom_logging import log
+from ..utils import ConfigManager
 
 class AlloyDBClient:
     """
@@ -35,11 +37,12 @@ class AlloyDBClient:
     """
 
     def __init__(self, 
-                 project_id: str, 
-                 region: str, 
-                 cluster_name:str, 
-                 instance_name:str, 
-                 user:str, 
+                 config:ConfigManager,
+                 project_id: str=None, 
+                 region: str=None, 
+                 cluster_name:str=None, 
+                 instance_name:str=None, 
+                 user:str=None, 
                  password=None, 
                  db="postgres"):
         """Initializes the AlloyDB client.
@@ -51,6 +54,23 @@ class AlloyDBClient:
          - password (str): The database user's password.
          - db_name (str): The name of the database.
         """
+        if config is None:
+            if project_id is None or region is None or cluster_name is None or instance_name is None:
+                raise ValueError("Must specify config or project_id, region, cluster_name, instance_name")
+        if config:
+            alloydb_config = config.vacConfig("alloydb_config")
+            if not alloydb_config:
+                raise ValueError("Must specify vac.alloydb_config")
+            project_id = alloydb_config["project_id"]
+            region = alloydb_config["region"]
+            cluster_name = alloydb_config["cluster"]
+            instance_name = alloydb_config["instance"]
+
+        ALLOYDB_DB = os.environ.get("ALLOYDB_DB")
+        if ALLOYDB_DB is None and alloydb_config.get("database") is None:
+            log.warning("Could not locate ALLOYDB_DB environment variable or 'alloydb_config.database'")
+        
+        self.database = alloydb_config.get("database") or ALLOYDB_DB,
         self.connector = Connector()
         self.inst_uri = self._build_instance_uri(project_id, region, cluster_name, instance_name)
         self.engine = self._create_engine(self.inst_uri, user, password, db)
@@ -100,6 +120,80 @@ class AlloyDBClient:
                 conn.close()
 
         return result
+    
+    async def execute_sql_async(self, sql_statement):
+        """Executes a given SQL statement asynchronously with error handling."""
+        sql_ = sqlalchemy.text(sql_statement)
+        result = None
+        async with self.engine.connect() as conn:
+            try:
+                log.info(f"Executing SQL statement asynchronously: {sql_}")
+                result = await conn.execute(sql_)
+            except DatabaseError as e:
+                if "already exists" in str(e):
+                    log.warning(f"Error ignored: {str(e)}. Assuming object already exists.")
+                else:
+                    raise
+            finally:
+                await conn.close()
+
+        return result
+
+    async def get_sources_from_docstore_async(self, sources, vector_name, search_type="OR", just_source_name=False):
+        """Fetches sources from the docstore asynchronously."""
+        if just_source_name:
+            query = self._list_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+        else:
+            query = self._get_sources_from_docstore(sources, vector_name=vector_name, search_type=search_type)
+
+        if not query:
+            return []
+
+        documents = await self.execute_sql_async(query)
+        return documents
+
+    def _get_sources_from_docstore(self, sources, vector_name, search_type="OR"):
+        """Helper function to build the SQL query for fetching sources."""
+        if not sources:
+            log.warning("No sources found for alloydb fetch")
+            return ""
+
+        table_name = f"{vector_name}_docstore"
+
+        conditions = self._and_or_ilike(sources, search_type=search_type)
+
+        query = f"""
+            SELECT * 
+            FROM {table_name}
+            WHERE {conditions}
+            ORDER BY langchain_metadata->>'objectId' ASC
+            LIMIT 500;
+        """
+
+        return query
+
+    def _list_sources_from_docstore(self, sources, vector_name, search_type="OR"):
+        """Helper function to build the SQL query for listing sources."""
+        table_name = f"{vector_name}_docstore"
+
+        if sources:
+            conditions = self._and_or_ilike(sources, search_type=search_type)
+            query = f"""
+                SELECT DISTINCT langchain_metadata->>'objectId' AS objectId
+                FROM {table_name}
+                WHERE {conditions}
+                ORDER BY langchain_metadata->>'objectId' ASC
+                LIMIT 500;
+            """
+        else:
+            query = f"""
+                SELECT DISTINCT langchain_metadata->>'objectId' AS objectId
+                FROM {table_name}
+                ORDER BY langchain_metadata->>'objectId' ASC
+                LIMIT 500;
+            """
+
+        return query
 
     @staticmethod
     def _and_or_ilike(sources, search_type="OR", operator="ILIKE"):

sunholo/discovery_engine/get_ai_search_chunks.py

@@ -43,7 +43,7 @@ def get_all_chunks(question:str, config:ConfigManager):
     return None
 
 def get_chunks(question, vector_name, num_chunks):
-    de = DiscoveryEngineClient(vector_name, project_id=get_gcp_project())
+    de = DiscoveryEngineClient(vector_name, project_id=get_gcp_project(include_config=True))
     try:
         return de.get_chunks(question, num_previous_chunks=num_chunks, num_next_chunks=num_chunks)
     except Exception as err:

sunholo/utils/config_class.py

@@ -37,9 +37,10 @@ class ConfigManager:
         self.config_folder = os.getenv("VAC_CONFIG_FOLDER", os.getcwd())
         self.local_config_folder = local_config_folder
         self.configs_by_kind = self.load_all_configs()
+        self.validate = validate
 
         test_agent = self.vacConfig("agent")
-        if not test_agent and self.vector_name != "global" and validate:
+        if not test_agent and self.vector_name != "global" and self.validate:
             print(f"WARNING: No vacConfig.agent found for {self.vector_name} - are you in right folder? {local_config_folder=} {self.config_folder=}")
 
     def load_all_configs(self):
@@ -125,7 +126,6 @@ class ConfigManager:
             else:
                 config = yaml.safe_load(file)
         self.config_cache[filename] = (config, datetime.now())
-        log.debug(f"Loaded and cached {config_file}")
         if is_local:
             log.warning(f"Local configuration override for {filename} via {self.local_config_folder}")
         return config

sunholo/utils/gcp_project.py

@@ -24,6 +24,7 @@ def get_gcp_project(include_config=False):
         gcp_config = load_config_key("gcp_config", "global", "vacConfig")
         if gcp_config:
             if gcp_config.get('project_id'):
+                logging.info("Using project_id from vacConfig.gcp_config.project_id")
                 return gcp_config.get('project_id')
 
     project_id = get_env_project_id()

{sunholo-0.80.6.dist-info → sunholo-0.81.0.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.80.6
+Version: 0.81.0
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.80.6.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.81.0.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.80.6.dist-info → sunholo-0.81.0.dist-info}/RECORD

@@ -55,11 +55,11 @@ sunholo/cli/swagger.py,sha256=absYKAU-7Yd2eiVNUY-g_WLl2zJfeRUNdWQ0oH8M_HM,1564
 sunholo/cli/vertex.py,sha256=8130YCarxHL1UC3aqblNmUwGZTXbkdL4Y_FOnZJsWiI,2056
 sunholo/components/__init__.py,sha256=IDoylb74zFKo6NIS3RQqUl0PDFBGVxM1dfUmO7OJ44U,176
 sunholo/components/llm.py,sha256=5wRVf7lIb7q1vRADNcdQp26L9l4vGHFIvjtUDurZN_s,11488
-sunholo/components/retriever.py,sha256=hKF3Az6DfDwUaKLkSTuzGg_5THS_lv7C51tfnzhb960,7653
+sunholo/components/retriever.py,sha256=bKIVT7_18Ut3OJd0E0jyiISPnD9qkHWVjcQPT4i1_G8,7720
 sunholo/components/vectorstore.py,sha256=xKk7micTRwZckaI7U6PxvFz_ZSjCH48xPTDYiDcv2tc,5913
-sunholo/database/__init__.py,sha256=Zz0Shcq-CtStf9rJGIYB_Ybzb8rY_Q9mfSj-nviM490,241
-sunholo/database/alloydb.py,sha256=c1PEmK9fJCxYaVmKv4emvOoXrajV7KqaVK5mqpeksvM,11527
-sunholo/database/alloydb_client.py,sha256=wJjBsGna0WT4-A_U-wP1UcHH-YAoCDkgX5dV2dCabKc,7261
+sunholo/database/__init__.py,sha256=bpB5Nk21kwqYj-qdVnvNgXjLsbflnH4g-San7OHMqR4,283
+sunholo/database/alloydb.py,sha256=YH8wNPS8gN-TDZEXQcVHxwd1NScHRfAxma3gK4R6KCk,11740
+sunholo/database/alloydb_client.py,sha256=vZu7m3n1TrGFqQ_F4kGgDEC-v_tnlJfByDGuI_4rrfw,11051
 sunholo/database/database.py,sha256=VqhZdkXUNdvWn8sUcUV3YNby1JDVf7IykPVXWBtxo9U,7361
 sunholo/database/lancedb.py,sha256=DyfZntiFKBlVPaFooNN1Z6Pl-LAs4nxWKKuq8GBqN58,715
 sunholo/database/static_dbs.py,sha256=8cvcMwUK6c32AS2e_WguKXWMkFf5iN3g9WHzsh0C07Q,442
@@ -74,7 +74,7 @@ sunholo/discovery_engine/__init__.py,sha256=P00bB8aVVWefOZbCQvzHsVMuP_sd-_d_4o5x
 sunholo/discovery_engine/chunker_handler.py,sha256=Fv4BLOBi_7ap3AiAy4TlTN48CLZSMurJ3TkvC75Euro,5123
 sunholo/discovery_engine/create_new.py,sha256=NzhSh6nG6nQ5J9gZh8IDph4JiEVT_DC5GGvP0GuwTWs,943
 sunholo/discovery_engine/discovery_engine_client.py,sha256=oORB2SVVqrYrz7E3srPrknyuR6Dl3SJJwaVrbVXJER4,17726
-sunholo/discovery_engine/get_ai_search_chunks.py,sha256=7yMpGaLU1nL3ttFLE-cIhNao-Vq9SY1edFC3T-7wH2I,1944
+sunholo/discovery_engine/get_ai_search_chunks.py,sha256=VPzdYoBP_E6Bko0KpX656QiIfJdwmje4sBnPtZs4JQ4,1963
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=MCbTePWjUbIRVDFFhHJ94BvOZvIom62-mTr0PmfQyt0,6951
 sunholo/gcs/__init__.py,sha256=SZvbsMFDko40sIRHTHppA37IijvJTae54vrhooEF5-4,90
@@ -118,10 +118,10 @@ sunholo/utils/__init__.py,sha256=Hv02T5L2zYWvCso5hzzwm8FQogwBq0OgtUbN_7Quzqc,89
 sunholo/utils/api_key.py,sha256=Ct4bIAQZxzPEw14hP586LpVxBAVi_W9Serpy0BK-7KI,244
 sunholo/utils/big_context.py,sha256=gJIP7_ZL-YSLhOMq8jmFTMqH1wq8eB1NK7oKPeZAq2s,5578
 sunholo/utils/config.py,sha256=aG29MXcL5qzQMtCMqcdy-2ysDCYf9Zn_ZLk5NNOQNSE,8982
-sunholo/utils/config_class.py,sha256=AbjTTgelMiaTPpTSBlbl6CC5wggcVE8ZFr4LJQ4t8Mg,9029
+sunholo/utils/config_class.py,sha256=3Hzeue_hvN1HR13c9VHCYUVspbvkLBpuIGvlGtxv_H8,9013
 sunholo/utils/config_schema.py,sha256=Wv-ncitzljOhgbDaq9qnFqH5LCuxNv59dTGDWgd1qdk,4189
 sunholo/utils/gcp.py,sha256=uueODEpA-P6O15-t0hmcGC9dONLO_hLfzSsSoQnkUss,4854
-sunholo/utils/gcp_project.py,sha256=0ozs6tzI4qEvEeXb8MxLnCdEVoWKxlM6OH05htj7_tc,1325
+sunholo/utils/gcp_project.py,sha256=Fa0IhCX12bZ1ctF_PKN8PNYd7hihEUfb90kilBfUDjg,1411
 sunholo/utils/parsers.py,sha256=akLSZLdvHf5T9OKVj5C3bo4g3Y8puATd8rxnbB4tWDs,5419
 sunholo/utils/timedelta.py,sha256=BbLabEx7_rbErj_YbNM0MBcaFN76DC4PTe4zD2ucezg,493
 sunholo/utils/user_ids.py,sha256=SQd5_H7FE7vcTZp9AQuQDWBXd4FEEd7TeVMQe1H4Ny8,292
@@ -134,9 +134,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=pgSahVDh7GPEulu3nl-w0jb5lTClb4TCnVxPnMokNZY,7533
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.80.6.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.80.6.dist-info/METADATA,sha256=CQEUn95SrlED6d_I0m_V-ZhLHTOXD0nVugPZbWl3N4k,7348
-sunholo-0.80.6.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-sunholo-0.80.6.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.80.6.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.80.6.dist-info/RECORD,,
+sunholo-0.81.0.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.81.0.dist-info/METADATA,sha256=jscapxO2wqyFdzlfqahL6VpisfqyNsETdfmHZwJR7I0,7348
+sunholo-0.81.0.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+sunholo-0.81.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.81.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.81.0.dist-info/RECORD,,