sunholo 0.126.3__py3-none-any.whl → 0.127.0__py3-none-any.whl

sunholo/agents/dispatch_to_qa.py

@@ -21,11 +21,6 @@ import traceback
 from .route import route_endpoint
 import os
 
-try:
-    from langfuse import Langfuse
-    langfuse = Langfuse()
-except ImportError:
-    langfuse = None
 
 def prep_request_payload(user_input, chat_history, vector_name, stream, **kwargs):
     """
@@ -93,7 +88,11 @@ def prep_request_payload(user_input, chat_history, vector_name, stream, **kwargs
     return qna_endpoint, qna_data
 
 def add_langfuse_trace(qna_endpoint):
-    if not langfuse:
+    try:
+        from langfuse import Langfuse
+        langfuse = Langfuse()
+    except Exception as err:
+        log.error(err)
         return None
     
     trace = langfuse.trace(name = f'dispatch/{os.path.basename(qna_endpoint)}')

sunholo/discovery_engine/chunker_handler.py

@@ -36,6 +36,7 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
             if vectorstore == "discovery_engine" or vectorstore == "vertex_ai_search":
                 log.info(f"Found vectorstore {vectorstore}")
                 if value.get('read_only'):
+                    log.info(f"{vectorstore} is read only, skipping")
                     continue
 
                 project_id = value.get("project_id")
@@ -50,6 +51,7 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
                 if not project_id:
                     raise ValueError("Couldn't retrieve project_id for vertex_ai_search")        
 
+                log.info(f"Using {project_id} and {location} for DiscoveryEngineClient")
                 corpus = DiscoveryEngineClient(
                     data_store_id=config.vector_name, 
                     project_id=project_id,
@@ -67,10 +69,13 @@ def do_discovery_engine(message_data:str, metadata:dict, config:ConfigManager=No
     if message_data.startswith("gs://"):
         log.info(f"DiscoveryEngineClient.import_files for {message_data}")
         if "/pdf_parts/" in message_data:
+            log.info(f"Not processing files with /pdf_parts/ - {message_data}")
             return None
         for corp in corpuses:
             try:
+                
                 metadata = audit_metadata(metadata, chunk_length=500)
+                log.info(f"Importing {message_data} {metadata=} to {corp}")
                 response = corp.import_document_with_metadata(
                     gcs_uri=message_data,
                     metadata=metadata
@@ -160,7 +165,10 @@ def discovery_engine_chunker_check(message_data,
         try:
             log.info(f"Process discovery engine for {metadata}")
             disc_meta = do_discovery_engine(message_data, metadata, config=config)
-            log.info(f"Processed discovery engine: {disc_meta}")
+            if disc_meta is None:
+                log.error(f"No disc_meta found for {metadata}")
+            else:
+                log.info(f"Processed discovery engine: {disc_meta}")
         except Exception as err:
             log.error(f"Error processing discovery engine: {str(err)} {traceback.format_exc()}")
             disc_meta = None

sunholo/discovery_engine/discovery_engine_client.py

@@ -16,6 +16,7 @@ import asyncio
 import json
 import uuid
 from ..utils.mime import guess_mime_type
+import traceback
 
 class DiscoveryEngineClient:
     """
@@ -80,8 +81,10 @@ class DiscoveryEngineClient:
             self.async_search_client = discoveryengine.SearchServiceAsyncClient(client_options=client_options)
         except RuntimeError:
             # No event loop in non-async environment, set async client to None
-            log.info("No event loop detected; skipping async client initialization")
+            log.info("No event loop detected; skipping Discoveryengine async client initialization")
             self.async_search_client = None
+        
+        log.info(f"Discovery Engine client initialized with {self.project_id=}, {self.data_store_id=}, {self.location=}")
 
     @classmethod
     def my_retry(cls):
@@ -490,6 +493,7 @@ class DiscoveryEngineClient:
             return doc_client.import_documents(request=request)
         
         try:
+            log.debug(f"Requesting import of documents: {request=}")
             operation = import_documents_with_retry(self.doc_client, request)
         except ResourceExhausted as e:
             log.error(f"DiscoveryEngine Operation failed after retries due to quota exceeded: {e}")
@@ -610,6 +614,7 @@ class DiscoveryEngineClient:
             reconciliation_mode=discoveryengine.ImportDocumentsRequest.ReconciliationMode.INCREMENTAL,
         )
 
+        log.debug(f"Making import_document_request: {request}")
         return self._import_document_request(request)
 
     def _create_unique_gsuri_docid(self, gcs_uri:str):
@@ -632,6 +637,7 @@ class DiscoveryEngineClient:
             str: The operation name.
         """
         try:
+            log.info(f"Importing doc with metadata: {gcs_uri=}, {metadata=}")
             # 1. Generate a unique document ID
             document_id = self._create_unique_gsuri_docid(gcs_uri)
 
@@ -662,7 +668,7 @@ class DiscoveryEngineClient:
             return self._import_document_request(request)
 
         except Exception as e:
-            log.error(f"Error importing document with metadata: {e}")
+            log.error(f"Error importing document with metadata: {e} {traceback.format_exc()}")
             raise e
     
     def get_mime_type(self, uri:str):

sunholo/embedder/embed_metadata.py

@@ -6,7 +6,7 @@ from ..utils.mime import guess_mime_type
 
 from ..custom_logging import log
 
-def audit_metadata(metadata, chunk_length=None):
+def audit_metadata(metadata:dict, chunk_length:int=None) -> dict:
     
     if 'eventTime' not in metadata:
         metadata['eventTime'] = datetime.datetime.now().isoformat(timespec='microseconds') + "Z"

sunholo/utils/config_class.py

@@ -112,18 +112,6 @@ class ConfigManager:
         """
         Helper function to load a config file and update the cache.
 
-        Args:
-            config_file (str): The path to the configuration file.
-            filename (str): The name of the configuration file.
-            is_local (bool): Indicates if the config file is from the local folder.
-
-        Returns:
-            dict: The loaded configuration.
-        """
-    def _reload_config_file(self, config_file, filename, is_local=False):
-        """
-        Helper function to load a config file and update the cache.
-
         Args:
             config_file (str): The path to the configuration file.
             filename (str): The name of the configuration file.
@@ -244,4 +232,23 @@ class ConfigManager:
             return agents[key]
         else:
             return agents.get("default")
+    
+    def permissionConfig(self, key: str):
+        """
+        Fetch a key from 'permissionConfig' kind configuration.
 
+        Args:
+            key (str): The key to fetch from the configuration.
+
+        Returns:
+            str: The value associated with the specified key.
+        """        
+        self._check_and_reload_configs()
+        config = self.configs_by_kind.get('permissionConfig')
+        if not config:
+            return None
+        agents = config.get('permissions')
+        if key in agents:
+            return agents[key]
+        else:
+            return None

{sunholo-0.126.3.dist-info → sunholo-0.127.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.126.3
+Version: 0.127.0
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0
@@ -142,6 +142,7 @@ Requires-Dist: google-cloud-discoveryengine>=0.13.4; extra == "gcp"
 Requires-Dist: google-cloud-texttospeech; extra == "gcp"
 Requires-Dist: google-genai>=0.2.2; extra == "gcp"
 Requires-Dist: google-generativeai>=0.8.3; extra == "gcp"
+Requires-Dist: langchain; extra == "gcp"
 Requires-Dist: langchain-google-genai>=2.0.0; extra == "gcp"
 Requires-Dist: langchain_google_alloydb_pg>=0.2.2; extra == "gcp"
 Requires-Dist: langchain-google-vertexai; extra == "gcp"
@@ -164,6 +165,7 @@ Requires-Dist: flask; extra == "http"
 Requires-Dist: gunicorn; extra == "http"
 Requires-Dist: httpcore; extra == "http"
 Requires-Dist: httpx; extra == "http"
+Requires-Dist: langchain; extra == "http"
 Requires-Dist: langfuse; extra == "http"
 Requires-Dist: python-socketio; extra == "http"
 Requires-Dist: requests; extra == "http"

{sunholo-0.126.3.dist-info → sunholo-0.127.0.dist-info}/RECORD

@@ -3,7 +3,7 @@ sunholo/custom_logging.py,sha256=YfIN1oP3dOEkkYkyRBU8BGS3uJFGwUDsFCl8mIVbwvE,122
 sunholo/langchain_types.py,sha256=uZ4zvgej_f7pLqjtu4YP7qMC_eZD5ym_5x4pyvA1Ih4,1834
 sunholo/agents/__init__.py,sha256=X2I3pPkGeKWjc3d0QgSpkTyqD8J8JtrEWqwrumf1MMc,391
 sunholo/agents/chat_history.py,sha256=Gph_CdlP2otYnNdR1q1Umyyyvcad2F6K3LxU5yBQ9l0,5387
-sunholo/agents/dispatch_to_qa.py,sha256=AwLS41oK6iS8xre-HuWjS4jj9dvU-evWI58EYfG65fg,8879
+sunholo/agents/dispatch_to_qa.py,sha256=NHihwAoCJ5_Lk11e_jZnucVUGQyZHCB-YpkfMHBCpQk,8882
 sunholo/agents/langserve.py,sha256=C46ph2mnygr6bdHijYWYyfQDI9ylAF0_9Kx2PfcCJpU,4414
 sunholo/agents/pubsub.py,sha256=TscZN_6am6DfaQkC-Yl18ZIBOoLE-0nDSiil6GpQEh4,1344
 sunholo/agents/route.py,sha256=mV8tGABbSqcg3PQL02MgQOs41gKEHLMyIJJJcTuFdbE,2988
@@ -72,14 +72,14 @@ sunholo/database/sql/sb/delete_source_row.sql,sha256=r6fEuUKdbiLHCDGKSbKINDCpJjs
 sunholo/database/sql/sb/return_sources.sql,sha256=89KAnxfK8n_qGK9jy1OQT8f9n4uYUtYL5cCxbC2mj_c,255
 sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3-zZI,89
 sunholo/discovery_engine/__init__.py,sha256=hLgqRDJ22Aov9o2QjAEfsVgnL3kMdM-g5p8RJ9OyKdQ,130
-sunholo/discovery_engine/chunker_handler.py,sha256=3KL33prRvN_fwaiZpOIXzxpKFUJdJiPQRZzhxYqunj0,7065
+sunholo/discovery_engine/chunker_handler.py,sha256=44qlTpdtz2GKzrhoQrxVMk-RPVFp7vQDPJoe9KmCcsw,7517
 sunholo/discovery_engine/cli.py,sha256=KGVle5rkLL49oF9TQhrGI--8017IvvLOEoYur545Qb0,12790
 sunholo/discovery_engine/create_new.py,sha256=WUi4_xh_dFaGX3xA9jkNKZhaR6LCELjMPeRb0hyj4FU,1226
-sunholo/discovery_engine/discovery_engine_client.py,sha256=NjIcP10I2-8yj6QZKrxGzNbh3SqQ5vGYsq9OwxCpWas,36935
+sunholo/discovery_engine/discovery_engine_client.py,sha256=0KhKRFKCvqvtkUOIrCXNk5353t9duuEtUQDhQnN2B24,37335
 sunholo/discovery_engine/get_ai_search_chunks.py,sha256=I6Dt1CznqEvE7XIZ2PkLqopmjpO96iVEWJJqL5cJjOU,5554
 sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
 sunholo/embedder/embed_chunk.py,sha256=did2pKkWM2o0KkRcb0H9l2x_WjCq6OyuHDxGbITFKPM,6530
-sunholo/embedder/embed_metadata.py,sha256=h9_L3Mkd7Mtnr8OwV4nNRrdSKoxhqh9LnSsht6j-vIY,6600
+sunholo/embedder/embed_metadata.py,sha256=qjv6oELuJRYKvR5SU5YHt-JAc_QfNOTNHbYeEXlQd1o,6617
 sunholo/excel/__init__.py,sha256=AqTMN9K4qJYi4maEgoORc5oxDVGO_eqmwzDaVP37JgY,56
 sunholo/excel/plugin.py,sha256=TJJdcKWyqEIce1agCJImvqvNp2CvLhzi4wUmLYHcLc8,4032
 sunholo/gcs/__init__.py,sha256=SZvbsMFDko40sIRHTHppA37IijvJTae54vrhooEF5-4,90
@@ -151,7 +151,7 @@ sunholo/utils/__init__.py,sha256=Hv02T5L2zYWvCso5hzzwm8FQogwBq0OgtUbN_7Quzqc,89
 sunholo/utils/api_key.py,sha256=Ct4bIAQZxzPEw14hP586LpVxBAVi_W9Serpy0BK-7KI,244
 sunholo/utils/big_context.py,sha256=HuP9_r_Nx1jvZHxjMEihgoZAXmnCh80zzsj1fq3mIOg,6021
 sunholo/utils/config.py,sha256=bz0ODJyqnoHQIsk4pmNpVxxq5WvwS0SfOq4cnCjQPJk,9105
-sunholo/utils/config_class.py,sha256=Z4sGzEkuxlCAJ8b-65_yzLmybnunywwOD9eXL8an5Wg,9619
+sunholo/utils/config_class.py,sha256=zhp71gNZb-t1cbkN2N3zwCAEiX0FSvKaKlpBxosgI4U,9750
 sunholo/utils/config_schema.py,sha256=Wv-ncitzljOhgbDaq9qnFqH5LCuxNv59dTGDWgd1qdk,4189
 sunholo/utils/gcp.py,sha256=lus1HH8YhFInw6QRKwfvKZq-Lz-2KQg4ips9v1I_3zE,4783
 sunholo/utils/gcp_project.py,sha256=Fa0IhCX12bZ1ctF_PKN8PNYd7hihEUfb90kilBfUDjg,1411
@@ -168,9 +168,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.126.3.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.126.3.dist-info/METADATA,sha256=_ekEbwlgG0P-PXQWNNfe9JnJDNDU_zXQ7ehPUNZprTM,10001
-sunholo-0.126.3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-sunholo-0.126.3.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.126.3.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.126.3.dist-info/RECORD,,
+sunholo-0.127.0.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.127.0.dist-info/METADATA,sha256=hvXLBki1RQ_ZLhcm3Ej-qQq1-9XADsZg-F6N7oxzP1A,10084
+sunholo-0.127.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+sunholo-0.127.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.127.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.127.0.dist-info/RECORD,,