sunholo 0.140.2__py3-none-any.whl → 0.140.5__py3-none-any.whl

sunholo/agents/__init__.py

@@ -2,6 +2,6 @@ from .chat_history import extract_chat_history
 from .dispatch_to_qa import send_to_qa, send_to_qa_async
 from .pubsub import process_pubsub
 from .special_commands import handle_special_commands, app_to_store, handle_files
-from .flask import register_qna_routes, create_app, VACRoutes
+from .flask import create_app, VACRoutes
 from .fastapi import register_qna_fastapi_routes, create_fastapi_app
 from .swagger import config_to_swagger

sunholo/agents/chat_history.py

@@ -1,5 +1,282 @@
 import json
 from ..custom_logging import log
+import time
+import hashlib
+from functools import lru_cache
+from typing import List, Tuple, Optional
+
+
+class ChatHistoryCache:
+    """
+    Incremental cache for chat history processing.
+    
+    Caches processed message pairs and only processes new messages
+    when the chat history is extended.
+    """
+    
+    def __init__(self, max_cache_size: int = 1000):
+        self.cache = {}
+        self.max_cache_size = max_cache_size
+    
+    def _get_cache_key(self, chat_history: List[dict]) -> str:
+        """Generate a cache key based on the chat history content."""
+        # Use the hash of the serialized chat history for the key
+        # Only hash the first few and last few messages to balance performance vs accuracy
+        if len(chat_history) <= 10:
+            content = str(chat_history)
+        else:
+            # Hash first 5 and last 5 messages + length
+            content = str(chat_history[:5] + chat_history[-5:] + [len(chat_history)])
+        
+        return hashlib.md5(content.encode()).hexdigest()
+    
+    def _find_cached_prefix(self, current_history: List[dict]) -> Tuple[Optional[List[Tuple]], int]:
+        """
+        Find the longest cached prefix of the current chat history.
+        
+        Returns:
+            Tuple of (cached_pairs, cache_length) or (None, 0) if no cache found
+        """
+        current_length = len(current_history)
+        
+        # Check for cached versions of prefixes, starting from longest
+        for cache_length in range(current_length - 1, 0, -1):
+            prefix = current_history[:cache_length]
+            cache_key = self._get_cache_key(prefix)
+            
+            if cache_key in self.cache:
+                cached_data = self.cache[cache_key]
+                cached_pairs = cached_data['pairs']
+                
+                # Verify the cache is still valid by checking a few messages
+                if self._verify_cache_validity(prefix, cached_data['original_history']):
+                    return cached_pairs, cache_length
+                else:
+                    # Cache is stale, remove it
+                    del self.cache[cache_key]
+        
+        return None, 0
+    
+    def _verify_cache_validity(self, current_prefix: List[dict], cached_prefix: List[dict]) -> bool:
+        """Quick verification that cached data is still valid."""
+        if len(current_prefix) != len(cached_prefix):
+            return False
+        
+        # Check first and last few messages for equality
+        check_indices = [0, -1] if len(current_prefix) >= 2 else [0]
+        
+        for i in check_indices:
+            if current_prefix[i] != cached_prefix[i]:
+                return False
+        
+        return True
+    
+    def extract_chat_history_incremental(self, chat_history: List[dict]) -> List[Tuple]:
+        """
+        Extract chat history with incremental caching.
+        
+        Args:
+            chat_history: List of chat message dictionaries
+            
+        Returns:
+            List of (human_message, ai_message) tuples
+        """
+        if not chat_history:
+            return []
+        
+        # Try to find cached prefix
+        cached_pairs, cache_length = self._find_cached_prefix(chat_history)
+        
+        if cached_pairs is not None:
+            log.debug(f"Found cached pairs for {cache_length} messages, processing {len(chat_history) - cache_length} new messages")
+            
+            # Process only the new messages
+            new_messages = chat_history[cache_length:]
+            new_pairs = self._process_new_messages(new_messages, cached_pairs)
+            
+            # Combine cached and new pairs
+            all_pairs = cached_pairs + new_pairs
+        else:
+            log.debug(f"No cache found, processing all {len(chat_history)} messages")
+            # Process all messages from scratch
+            all_pairs = self._extract_chat_history_full(chat_history)
+        
+        # Cache the result
+        self._update_cache(chat_history, all_pairs)
+        
+        return all_pairs
+    
+    def _process_new_messages(self, new_messages: List[dict], cached_pairs: List[Tuple]) -> List[Tuple]:
+        """
+        Process only the new messages, considering the state from cached pairs.
+        
+        Args:
+            new_messages: New messages to process
+            cached_pairs: Previously processed message pairs
+            
+        Returns:
+            List of new message pairs
+        """
+        if not new_messages:
+            return []
+        
+        new_pairs = []
+        
+        # Determine if we're waiting for a bot response based on cached pairs
+        waiting_for_bot = True
+        if cached_pairs:
+            last_pair = cached_pairs[-1]
+            # If last pair has both human and AI message, we're ready for a new human message
+            waiting_for_bot = not (last_pair[0] and last_pair[1])
+        
+        # If we ended with an unpaired human message, get it
+        last_human_message = ""
+        if cached_pairs and waiting_for_bot:
+            last_human_message = cached_pairs[-1][0]
+        
+        # Process new messages
+        for message in new_messages:
+            try:
+                is_human_msg = is_human(message)
+                content = create_message_element(message)
+                
+                if is_human_msg:
+                    last_human_message = content
+                    waiting_for_bot = True
+                else:  # Bot message
+                    if waiting_for_bot and last_human_message:
+                        new_pairs.append((last_human_message, content))
+                        last_human_message = ""
+                        waiting_for_bot = False
+                    # If not waiting for bot or no human message, this is an orphaned bot message
+                    
+            except (KeyError, TypeError) as e:
+                log.warning(f"Error processing new message: {e}")
+                continue
+        
+        return new_pairs
+    
+    def _extract_chat_history_full(self, chat_history: List[dict]) -> List[Tuple]:
+        """Full extraction when no cache is available."""
+        # Use the optimized version from before
+        paired_messages = []
+        
+        # Handle initial bot message
+        start_idx = 0
+        if chat_history and is_bot(chat_history[0]):
+            try:
+                first_message = chat_history[0]
+                blank_element = ""
+                bot_element = create_message_element(first_message)
+                paired_messages.append((blank_element, bot_element))
+                start_idx = 1
+            except (KeyError, TypeError):
+                pass
+        
+        # Process remaining messages
+        last_human_message = ""
+        for i in range(start_idx, len(chat_history)):
+            message = chat_history[i]
+            
+            try:
+                is_human_msg = is_human(message)
+                content = create_message_element(message)
+                
+                if is_human_msg:
+                    last_human_message = content
+                else:  # Bot message
+                    if last_human_message:
+                        paired_messages.append((last_human_message, content))
+                        last_human_message = ""
+                        
+            except (KeyError, TypeError) as e:
+                log.warning(f"Error processing message {i}: {e}")
+                continue
+        
+        return paired_messages
+    
+    def _update_cache(self, chat_history: List[dict], pairs: List[Tuple]):
+        """Update cache with new result."""
+        # Only cache if the history is of reasonable size
+        if len(chat_history) < 2:
+            return
+        
+        cache_key = self._get_cache_key(chat_history)
+        
+        # Implement simple LRU by removing oldest entries
+        if len(self.cache) >= self.max_cache_size:
+            # Remove 20% of oldest entries
+            remove_count = self.max_cache_size // 5
+            oldest_keys = list(self.cache.keys())[:remove_count]
+            for key in oldest_keys:
+                del self.cache[key]
+        
+        self.cache[cache_key] = {
+            'pairs': pairs,
+            'original_history': chat_history.copy(),  # Store copy for validation
+            'timestamp': time.time()
+        }
+        
+        log.debug(f"Cached {len(pairs)} pairs for history of length {len(chat_history)}")
+    
+    def clear_cache(self):
+        """Clear the entire cache."""
+        self.cache.clear()
+        log.info("Chat history cache cleared")
+
+
+# Global cache instance
+_chat_history_cache = ChatHistoryCache()
+
+
+def extract_chat_history_with_cache(chat_history: List[dict] = None) -> List[Tuple]:
+    """
+    Main function to replace the original extract_chat_history.
+    
+    Uses incremental caching for better performance with growing chat histories.
+    """
+    if not chat_history:
+        log.debug("No chat history found")
+        return []
+    
+    return _chat_history_cache.extract_chat_history_incremental(chat_history)
+
+
+# Async version that wraps the cached version
+async def extract_chat_history_async_cached(chat_history: List[dict] = None) -> List[Tuple]:
+    """
+    Async version that uses the cache and runs in a thread pool if needed.
+    """
+    import asyncio
+    
+    if not chat_history:
+        return []
+    
+    # For very large histories, run in thread pool to avoid blocking
+    if len(chat_history) > 1000:
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(
+            None, 
+            extract_chat_history_with_cache, 
+            chat_history
+        )
+    else:
+        # For smaller histories, just run directly
+        return extract_chat_history_with_cache(chat_history)
+
+
+# Utility function to warm up the cache
+def warm_up_cache(chat_histories: List[List[dict]]):
+    """
+    Pre-populate cache with common chat histories.
+    
+    Args:
+        chat_histories: List of chat history lists to cache
+    """
+    for history in chat_histories:
+        extract_chat_history_with_cache(history)
+    
+    log.info(f"Warmed up cache with {len(chat_histories)} chat histories")
 
 
 async def extract_chat_history_async(chat_history=None):
@@ -243,3 +520,4 @@ def is_ai(message: dict):
         return message['role'] == 'assistant'
     else:
         return 'bot_id' in message  # Slack
+

sunholo/agents/flask/__init__.py

@@ -1,3 +1,2 @@
-from .qna_routes import register_qna_routes
 from .base import create_app
 from .vac_routes import VACRoutes

sunholo/agents/flask/vac_routes.py

@@ -7,8 +7,8 @@ from functools import partial
 import inspect
 import asyncio
 
-from ...agents import extract_chat_history, handle_special_commands
-from ..chat_history import extract_chat_history_async
+from ...agents import handle_special_commands
+from ..chat_history import extract_chat_history_with_cache, extract_chat_history_async_cached
 from ...qna.parsers import parse_output
 from ...streaming import start_streaming_chat, start_streaming_chat_async
 from ...archive import archive_qa
@@ -58,12 +58,18 @@ if __name__ == "__main__":
 ```
     
     """
-    def __init__(self, app, stream_interpreter: callable, vac_interpreter:callable=None, additional_routes:dict=None, async_stream:bool=False):
+    def __init__(self, app, 
+                 stream_interpreter: callable, 
+                 vac_interpreter:callable=None, 
+                 additional_routes:dict=None, 
+                 async_stream:bool=False,
+                 add_langfuse_eval:bool=True):
         self.app = app
         self.stream_interpreter = stream_interpreter
         self.vac_interpreter = vac_interpreter or partial(self.vac_interpreter_default)
         self.additional_routes = additional_routes if additional_routes is not None else []
         self.async_stream = async_stream
+        self.add_langfuse_eval = add_langfuse_eval
         self.register_routes()
         
 
@@ -96,13 +102,9 @@ if __name__ == "__main__":
         # Basic routes
         self.app.route("/", methods=['GET'])(self.home)
         self.app.route("/health", methods=['GET'])(self.health)
-
-        # Streaming VAC
-        self.app.route('/vac/streaming/<vector_name>', 
-                       methods=['POST'], 
-                       provide_automatic_options=False)(self.handle_stream_vac)
         
         if self.async_stream:  # Use async treatment
+            log.info("async_stream enabled")
             self.app.route('/vac/streaming/<vector_name>', 
                         methods=['POST'], 
                         provide_automatic_options=False)(self.handle_stream_vac_async)
@@ -351,10 +353,10 @@ if __name__ == "__main__":
 
         # Use the async version of prep_vac
         prep = await self.prep_vac_async(request, vector_name)
-        log.info(f"Processing prep: {prep}")
+        log.info(f"Processing async prep: {prep}")
         all_input = prep["all_input"]
 
-        log.info(f'Streaming data with: {all_input}')
+        log.info(f'Streaming async data with: {all_input}')
 
         async def generate_response_content():
             try:
@@ -378,12 +380,12 @@ if __name__ == "__main__":
                         yield chunk
 
             except Exception as e:
-                yield f"Streaming Error: {str(e)} {traceback.format_exc()}"
+                yield f"Streaming async Error: {str(e)} {traceback.format_exc()}"
 
         response = Response(generate_response_content(), content_type='text/plain; charset=utf-8')
         response.headers['Transfer-Encoding'] = 'chunked'
 
-        log.debug(f"streaming response: {response}")
+        log.debug(f"streaming async response: {response}")
 
         return response
 
@@ -554,7 +556,8 @@ if __name__ == "__main__":
         else:
             log.info(f"User message: {user_message}")
     
-        paired_messages = extract_chat_history(chat_history)
+        paired_messages = extract_chat_history_with_cache(chat_history)
+
         command_response = handle_special_commands(user_message, vector_name, paired_messages)
 
         if command_response is not None:
@@ -698,10 +701,10 @@ if __name__ == "__main__":
 
         trace = None
         span = None
-
-        trace_id = data.get('trace_id')
-        trace = self.create_langfuse_trace(request, vector_name, trace_id)
-        log.info(f"Using existing langfuse trace: {trace_id}")
+        if self.add_langfuse_eval:
+            trace_id = data.get('trace_id')
+            trace = self.create_langfuse_trace(request, vector_name, trace_id)
+            log.info(f"Using existing langfuse trace: {trace_id}")
         
         #config, _ = load_config("config/llm_config.yaml")
         try:
@@ -725,7 +728,7 @@ if __name__ == "__main__":
         vector_name = data.pop('vector_name', vector_name)
         data.pop('trace_id', None) # to ensure not in kwargs
 
-        paired_messages = extract_chat_history(chat_history)
+        paired_messages = extract_chat_history_with_cache(chat_history)
 
         all_input = {'user_input': user_input, 
                      'vector_name': vector_name, 
@@ -741,15 +744,10 @@ if __name__ == "__main__":
                 metadata=vac_config.configs_by_kind,
                 input = all_input
             )
-        command_response = handle_special_commands(user_input, vector_name, paired_messages)
-        if command_response is not None:
-            if trace:
-                trace.update(output=jsonify(command_response))
     
         return {
             "trace": trace,
             "span": span,
-            "command_response": command_response,
             "all_input": all_input,
             "vac_config": vac_config
         }
@@ -793,7 +791,7 @@ if __name__ == "__main__":
         data.pop('trace_id', None)  # to ensure not in kwargs
         
         # Task 3: Process chat history
-        chat_history_task = asyncio.create_task(extract_chat_history_async(chat_history))
+        chat_history_task = asyncio.create_task(extract_chat_history_async_cached(chat_history))
         tasks.append(chat_history_task)
         
         # Await all tasks concurrently

{sunholo-0.140.2.dist-info → sunholo-0.140.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.140.2
+Version: 0.140.5
 Summary: AI DevOps - a package to help deploy GenAI to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.140.2.dist-info → sunholo-0.140.5.dist-info}/RECORD

@@ -1,8 +1,8 @@
 sunholo/__init__.py,sha256=InRbX4V0-qdNHo9zYH3GEye7ASLR6LX8-SMvPV4Jsaw,1212
 sunholo/custom_logging.py,sha256=JXZTnXp_DixP3jwYfKw4LYRDS9IuTq7ctCgfZbI2rxA,22023
 sunholo/langchain_types.py,sha256=uZ4zvgej_f7pLqjtu4YP7qMC_eZD5ym_5x4pyvA1Ih4,1834
-sunholo/agents/__init__.py,sha256=X2I3pPkGeKWjc3d0QgSpkTyqD8J8JtrEWqwrumf1MMc,391
-sunholo/agents/chat_history.py,sha256=4jGCHBP8dZfUjSJPxgKyh6nOqhnHRn1x9U3CnGb0I5E,7624
+sunholo/agents/__init__.py,sha256=AauG3l0y4r5Fzx1zJfZ634M4o-0o7B7J5T8k_gPvNqE,370
+sunholo/agents/chat_history.py,sha256=e2NmiooaRUxKGr_aoU05rzhHi3VsKjbZZmzeDr2yJJE,17780
 sunholo/agents/dispatch_to_qa.py,sha256=NHihwAoCJ5_Lk11e_jZnucVUGQyZHCB-YpkfMHBCpQk,8882
 sunholo/agents/langserve.py,sha256=C46ph2mnygr6bdHijYWYyfQDI9ylAF0_9Kx2PfcCJpU,4414
 sunholo/agents/pubsub.py,sha256=TscZN_6am6DfaQkC-Yl18ZIBOoLE-0nDSiil6GpQEh4,1344
@@ -12,10 +12,9 @@ sunholo/agents/swagger.py,sha256=2tzGmpveUMmTREykZvVnDj3j295wyOMu7mUFDnXdY3c,106
 sunholo/agents/fastapi/__init__.py,sha256=S_pj4_bTUmDGoq_exaREHlOKThi0zTuGT0VZY0YfODQ,88
 sunholo/agents/fastapi/base.py,sha256=W-cyF8ZDUH40rc-c-Apw3-_8IIi2e4Y9qRtnoVnsc1Q,2521
 sunholo/agents/fastapi/qna_routes.py,sha256=lKHkXPmwltu9EH3RMwmD153-J6pE7kWQ4BhBlV3to-s,3864
-sunholo/agents/flask/__init__.py,sha256=poJDKMr2qj8qMb99JqCvCPSiEt1tj2tLQ3hKW3f2aVw,107
+sunholo/agents/flask/__init__.py,sha256=dEoByI3gDNUOjpX1uVKP7uPjhfFHJubbiaAv3xLopnk,63
 sunholo/agents/flask/base.py,sha256=HLz3Z5efWaewTwSFEM6JH48NA9otoJBoVFJlARGk9L8,788
-sunholo/agents/flask/qna_routes.py,sha256=uwUD1yrzOPH27m2AXpiQrPk_2VfJOQOM6dAynOWQtoQ,22532
-sunholo/agents/flask/vac_routes.py,sha256=RprhFJje5gTNU3ePGbCCPdBAdYs417VyXNlGe5UlR-g,33370
+sunholo/agents/flask/vac_routes.py,sha256=rpakKOO6bjBJv2l0NOqd0fAzRJEFLR1buSN8GiubMeE,33230
 sunholo/archive/__init__.py,sha256=qNHWm5rGPVOlxZBZCpA1wTYPbalizRT7f8X4rs2t290,31
 sunholo/archive/archive.py,sha256=PxVfDtO2_2ZEEbnhXSCbXLdeoHoQVImo4y3Jr2XkCFY,1204
 sunholo/auth/__init__.py,sha256=TeP-OY0XGxYV_8AQcVGoh35bvyWhNUcMRfhuD5l44Sk,91
@@ -169,9 +168,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.140.2.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.140.2.dist-info/METADATA,sha256=nMq7_Z9VKq4kPXXTvyeirWinNf5tEMmUe0HnIiFhYAc,10067
-sunholo-0.140.2.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-sunholo-0.140.2.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.140.2.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.140.2.dist-info/RECORD,,
+sunholo-0.140.5.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.140.5.dist-info/METADATA,sha256=SM-i1Mdu3Sl5-m2am-EwU1DTAv9M22QZIylaV9MXkPg,10067
+sunholo-0.140.5.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+sunholo-0.140.5.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.140.5.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.140.5.dist-info/RECORD,,

sunholo/agents/flask/qna_routes.py

@@ -1,604 +0,0 @@
-#   Copyright [2024] [Holosun ApS]
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-
-import json
-import traceback
-import uuid
-
-from ...agents import extract_chat_history, handle_special_commands
-from ...qna.parsers import parse_output
-from ...streaming import start_streaming_chat
-from ...archive import archive_qa
-from ...custom_logging import log
-from ...utils.config import load_config
-from ...utils import ConfigManager
-from ...utils.version import sunholo_version
-import os
-from ...gcs.add_file import add_file_to_gcs, handle_base64_image
-from ..swagger import validate_api_key
-from datetime import datetime, timedelta
-
-try:
-    from flask import request, jsonify, Response
-except ImportError:
-    pass
-
-try:
-    from langfuse.decorators import langfuse_context, observe
-except ImportError:
-    pass    
-
-# Cache dictionary to store validated API keys
-api_key_cache = {}
-cache_duration = timedelta(minutes=5)  # Cache duration
-
-def make_openai_response(user_message, vector_name, answer):
-    response_id = str(uuid.uuid4())
-    log.info(f"openai response: Q: {user_message} to VECTOR_NAME: {vector_name} - A: {answer}")
-    openai_response = {
-        "id": response_id,
-        "object": "chat.completion",
-        "created": str(int(datetime.now().timestamp())),
-        "model": vector_name,
-        "system_fingerprint": sunholo_version(),
-        "choices": [{
-            "index": 0,
-            "message": {
-                "role": "assistant",
-                "content": answer,
-            },
-            "logprobs": None,
-            "finish_reason": "stop"
-        }],
-        "usage": {
-            "prompt_tokens": 0,
-            "completion_tokens": 0,
-            "total_tokens": 0
-        }
-    }
-
-    log.info(f"OpenAI response: {openai_response}")
-    return jsonify(openai_response)
-
-def register_qna_routes(app, stream_interpreter, vac_interpreter):
-    """
-    Register Q&A routes for a Flask application.
-
-    This function sets up multiple routes for handling Q&A operations,
-    including streaming responses and processing static responses.
-
-    Args:
-        app (Flask): The Flask application instance.
-        stream_interpreter (function): Function to handle streaming Q&A responses.
-        vac_interpreter (function): Function to handle static Q&A responses.
-
-    Returns:
-        None
-
-    Example:
-        from flask import Flask
-        app = Flask(__name__)
-
-        def dummy_stream_interpreter(...):
-            ...
-
-        def dummy_vac_interpreter(...):
-            ...
-
-        register_qna_routes(app, dummy_stream_interpreter, dummy_vac_interpreter)
-    """
-    @app.route("/")
-    def home():
-        return jsonify("OK")
-
-    @app.route("/health")
-    def health():
-        return jsonify({"status": "healthy"})
-
-    @app.route('/vac/streaming/<vector_name>', methods=['POST'])
-    def stream_qa(vector_name):
-        """
-        Handle streaming Q&A responses.
-
-        This function sets up a route to handle streaming Q&A responses based on
-        the provided vector name.
-
-        Args:
-            vector_name (str): The name of the vector for the request.
-
-        Returns:
-            Response: A Flask response object streaming the Q&A response content.
-
-        Example:
-            response = stream_qa("example_vector")
-        """
-        observed_stream_interpreter = observe()(stream_interpreter)
-        prep = prep_vac(request, vector_name)
-        log.debug(f"Processing prep: {prep}")
-        trace = prep["trace"]
-        span = prep["span"]
-        command_response = prep["command_response"]
-        vac_config = prep["vac_config"]
-        all_input = prep["all_input"]
-
-        if command_response:
-            return jsonify(command_response)
-
-        log.info(f'Streaming data with: {all_input}')
-        if span:
-            generation = span.generation(
-                name="start_streaming_chat",
-                metadata=vac_config,
-                input = all_input,
-                completion_start_time=datetime.now(),
-                model=vac_config.get("model") or vac_config.get("llm")
-            )
-
-        def generate_response_content():
-
-            for chunk in start_streaming_chat(question=all_input["user_input"],
-                                              vector_name=vector_name,
-                                              qna_func=observed_stream_interpreter,
-                                              chat_history=all_input["chat_history"],
-                                              wait_time=all_input["stream_wait_time"],
-                                              timeout=all_input["stream_timeout"],
-                                              #kwargs
-                                              **all_input["kwargs"]
-                                              ):
-                if isinstance(chunk, dict) and 'answer' in chunk:
-                    # When we encounter the dictionary, we yield it as a JSON string
-                    # and stop the generator.
-                    if trace:
-                        chunk["trace"] = trace.id
-                        chunk["trace_url"] = trace.get_trace_url()
-                    archive_qa(chunk, vector_name)
-                    if trace:
-                        generation.end(output=json.dumps(chunk))
-                        span.end(output=json.dumps(chunk))
-                        trace.update(output=json.dumps(chunk))
-  
-                    return json.dumps(chunk)
-                
-                else:
-                    # Otherwise, we yield the plain text chunks as they come in.
-                    yield chunk
-            
-        # Here, the generator function will handle streaming the content to the client.
-        response = Response(generate_response_content(), content_type='text/plain; charset=utf-8')
-        response.headers['Transfer-Encoding'] = 'chunked'  
-
-        log.debug(f"streaming response: {response}")
-        if trace:
-            generation.end(output=response)
-            span.end(output=response)
-            trace.update(output=response)
-        
-        #if 'user_id' in all_input["kwargs"]:
-        #    kwargs = all_input["kwargs"]
-        #    config = ConfigManager(vector_name)
-        #    add_user_history_rag(kwargs.pop('user_id'), 
-        #                         config, 
-        #                         question=all_input.pop("user_input"), 
-        #                         answer=response.get('answer'), 
-        #                         metadata=all_input)
-
-        return response
-
-    @app.route('/vac/<vector_name>', methods=['POST'])
-    def process_qna(vector_name):
-        """
-        Handle static Q&A responses.
-
-        This function sets up a route to handle static Q&A responses based on
-        the provided vector name.
-
-        Args:
-            vector_name (str): The name of the vector for the request.
-
-        Returns:
-            Response: A Flask response object with the Q&A response content.
-
-        Example:
-            response = process_qna("example_vector")
-        """
-        observed_vac_interpreter = observe()(vac_interpreter)
-        prep = prep_vac(request, vector_name)
-        log.debug(f"Processing prep: {prep}")
-        trace = prep["trace"]
-        span = prep["span"]
-        command_response = prep["command_response"]
-        vac_config = prep["vac_config"]
-        all_input = prep["all_input"]
-
-        if command_response:
-            return jsonify(command_response)
-
-        try:
-            if span:
-                generation = span.generation(
-                    name="vac_interpreter",
-                    metadata=vac_config,
-                    input = all_input,
-                    model=vac_config.get("model") or vac_config.get("llm")
-                )
-            bot_output = observed_vac_interpreter(
-                question=all_input["user_input"],
-                vector_name=vector_name,
-                chat_history=all_input["chat_history"],
-                **all_input["kwargs"]
-            )
-            if span:
-                generation.end(output=bot_output)
-            # {"answer": "The answer", "source_documents": [{"page_content": "The page content", "metadata": "The metadata"}]}
-            bot_output = parse_output(bot_output)
-            if trace:
-                bot_output["trace"] = trace.id
-                bot_output["trace_url"] = trace.get_trace_url()
-            archive_qa(bot_output, vector_name)
-            log.info(f'==LLM Q:{all_input["user_input"]} - A:{bot_output}')
-
-
-        except Exception as err: 
-            bot_output = {'answer': f'QNA_ERROR: An error occurred while processing /vac/{vector_name}: {str(err)} traceback: {traceback.format_exc()}'}
-        
-        if trace:
-            span.end(output=jsonify(bot_output))
-            trace.update(output=jsonify(bot_output)) 
-
-        # {'answer': 'output'}
-        return jsonify(bot_output)
-
-    @app.before_request
-    def check_authentication_header():
-        if request.path.startswith('/openai/'):
-            log.debug(f'Request headers: {request.headers}')
-            # the header forwarded
-            auth_header = request.headers.get('X-Forwarded-Authorization')
-            if auth_header:
-                
-                if auth_header.startswith('Bearer '):
-                    api_key = auth_header.split(' ')[1]  # Assuming "Bearer <api_key>"
-                else:
-                    return jsonify({'error': 'Invalid authorization header does not start with "Bearer " - got: {auth_header}'}), 401
-                
-                endpoints_host = os.getenv('_ENDPOINTS_HOST')
-                if not endpoints_host:
-                    return jsonify({'error': '_ENDPOINTS_HOST environment variable not found'}), 401
-                
-                # Check cache first
-                current_time = datetime.now()
-                if api_key in api_key_cache:
-                    cached_result, cache_time = api_key_cache[api_key]
-                    if current_time - cache_time < cache_duration:
-                        if not cached_result:
-                            return jsonify({'error': 'Invalid cached API key'}), 401
-                        else:
-                            return  # Valid API key, continue to the endpoint
-                    else:
-                        # Cache expired, remove from cache
-                        del api_key_cache[api_key]
-                
-                # Validate API key
-                is_valid = validate_api_key(api_key, endpoints_host)
-                # Update cache
-                api_key_cache[api_key] = (is_valid, current_time)
-                
-                if not is_valid:
-                    return jsonify({'error': 'Invalid API key'}), 401
-            else:
-                return jsonify({'error': 'Missing Authorization header'}), 401
-
-    @app.route('/openai/health', methods=['GET', 'POST'])
-    def openai_health_endpoint():
-        return jsonify({'message': 'Success'})
-
-    @app.route('/openai/v1/chat/completions', methods=['POST'])
-    @app.route('/openai/v1/chat/completions/<vector_name>', methods=['POST'])
-    def openai_compatible_endpoint(vector_name=None):
-        """
-        Handle OpenAI-compatible chat completions.
-
-        This function sets up routes to handle OpenAI-compatible chat completion requests,
-        both with and without a specified vector name.
-
-        Args:
-            vector_name (str, optional): The name of the vector for the request. Defaults to None.
-
-        Returns:
-            Response: A Flask response object with the chat completion content.
-
-        Example:
-            response = openai_compatible_endpoint("example_vector")
-        """
-        data = request.get_json()
-        log.info(f'openai_compatible_endpoint got data: {data} for vector: {vector_name}')
-
-        vector_name = vector_name or data.pop('model', None)
-        messages = data.pop('messages', None)
-        chat_history = data.pop('chat_history', None)
-        stream = data.pop('stream', False)
-
-        if not messages:
-            return jsonify({"error": "No messages provided"}), 400
-
-        user_message = None
-        image_uri = None
-        mime_type = None
-
-        for msg in reversed(messages):
-            if msg['role'] == 'user':
-                if isinstance(msg['content'], list):
-                    for content_item in msg['content']:
-                        if content_item['type'] == 'text':
-                            user_message = content_item['text']
-                        elif content_item['type'] == 'image_url':
-                            base64_data = content_item['image_url']['url']
-                            image_uri, mime_type = handle_base64_image(base64_data, vector_name)
-                else:
-                    user_message = msg['content']
-                break
-
-        if not user_message:
-            return jsonify({"error": "No user message provided"}), 400
-        else:
-            log.info(f"User message: {user_message}")
-        
-        paired_messages = extract_chat_history(chat_history)
-        command_response = handle_special_commands(user_message, vector_name, paired_messages)
-
-        if command_response is not None:
-
-            return make_openai_response(user_message, vector_name, command_response)
-        
-        if image_uri:
-            data["image_uri"] = image_uri
-            data["mime"] = mime_type
-
-        all_input = {
-            "user_input": user_message,
-            "chat_history": chat_history,
-            "kwargs": data
-        }
-
-        observed_stream_interpreter = observe()(stream_interpreter)
-
-        response_id = str(uuid.uuid4())
-        
-        def generate_response_content():
-            for chunk in start_streaming_chat(question=user_message,
-                                            vector_name=vector_name,
-                                            qna_func=observed_stream_interpreter,
-                                            chat_history=all_input["chat_history"],
-                                            wait_time=all_input.get("stream_wait_time", 1),
-                                            timeout=all_input.get("stream_timeout", 60),
-                                            **all_input["kwargs"]
-                                            ):
-                if isinstance(chunk, dict) and 'answer' in chunk:
-                    openai_chunk = {
-                        "id": response_id,
-                        "object": "chat.completion.chunk",
-                        "created": str(int(datetime.now().timestamp())),
-                        "model": vector_name,
-                        "system_fingerprint": sunholo_version(),
-                        "choices": [{
-                            "index": 0,
-                            "delta": {"content": chunk['answer']},
-                            "logprobs": None,
-                            "finish_reason": None
-                        }]
-                    }
-                    yield json.dumps(openai_chunk) + "\n"
-                else:
-                    log.info(f"Unknown chunk: {chunk}")
-
-            final_chunk = {
-                "id": response_id,
-                "object": "chat.completion.chunk",
-                "created": str(int(datetime.now().timestamp())),
-                "model": vector_name,
-                "system_fingerprint": sunholo_version(),
-                "choices": [{
-                    "index": 0,
-                    "delta": {},
-                    "logprobs": None,
-                    "finish_reason": "stop"
-                }]
-            }
-            yield json.dumps(final_chunk) + "\n"
-
-        if stream:
-            log.info("Streaming openai chunks")
-            return Response(generate_response_content(), content_type='text/plain; charset=utf-8')
-
-        try:
-            observed_vac_interpreter = observe()(vac_interpreter)
-            bot_output = observed_vac_interpreter(
-                question=user_message,
-                vector_name=vector_name,
-                chat_history=all_input["chat_history"],
-                **all_input["kwargs"]
-            )
-            bot_output = parse_output(bot_output)
-
-            log.info(f"Bot output: {bot_output}")
-            if bot_output:
-                return make_openai_response(user_message, vector_name, bot_output.get('answer', ''))
-            else:
-                return make_openai_response(user_message, vector_name, 'ERROR: could not find an answer')
-
-        except Exception as err:
-            log.error(f"OpenAI response error: {str(err)} traceback: {traceback.format_exc()}")
-            
-            return make_openai_response(user_message, vector_name, f'ERROR: {str(err)}')
-
-
-def create_langfuse_trace(request, vector_name):
-    """
-    Create a Langfuse trace for tracking requests.
-
-    This function initializes a Langfuse trace object based on the request headers
-    and vector name.
-
-    Args:
-        request (Request): The Flask request object.
-        vector_name (str): The name of the vector for the request.
-
-    Returns:
-        Langfuse.Trace: The Langfuse trace object.
-
-    Example:
-        trace = create_langfuse_trace(request, "example_vector")
-    """
-    try:
-        from langfuse import Langfuse
-        langfuse = Langfuse()
-    except ImportError as err:
-        print(f"No langfuse installed for agents.flask.register_qna_routes, install via `pip install sunholo[http]` - {str(err)}")
-        
-        return None
-
-    user_id = request.headers.get("X-User-ID")
-    session_id = request.headers.get("X-Session-ID")
-    message_source = request.headers.get("X-Message-Source")
-
-    package_version = sunholo_version()
-    tags = [package_version]
-    if message_source:
-        tags.append(message_source)
-
-    return langfuse.trace(
-        name = f"/vac/{vector_name}",
-        user_id = user_id,
-        session_id = session_id,
-        tags = tags,
-        release = f"sunholo-v{package_version}"
-    )
-
-def prep_vac(request, vector_name):
-    """
-    Prepare the input data for a VAC request.
-
-    This function processes the incoming request data, extracts relevant
-    information, and prepares the data for VAC processing.
-
-    Args:
-        request (Request): The Flask request object.
-        vector_name (str): The name of the vector for the request.
-
-    Returns:
-        dict: A dictionary containing prepared input data and metadata.
-
-    Example:
-        prep_data = prep_vac(request, "example_vector")
-    """
-    #trace = create_langfuse_trace(request, vector_name)
-    trace = None
-    span = None
-
-    if request.content_type.startswith('application/json'):
-        data = request.get_json()
-    elif request.content_type.startswith('multipart/form-data'):
-        data = request.form.to_dict()
-        if 'file' in request.files:
-            file = request.files['file']
-            if file.filename != '':
-                log.info(f"Found file: {file.filename} to upload to GCS")
-                try:
-                    image_uri, mime_type = handle_file_upload(file, vector_name)
-                    data["image_uri"] = image_uri
-                    data["image_url"] = image_uri
-                    data["mime"] = mime_type
-                except Exception as e:
-                    log.error(f"Error uploading file: {str(e)}")
-            else:
-                log.info("No file selected to upload to GCS")
-    else:
-        log.warning(f"Error uploading file: Unsupported content type {request.content_type}")
-
-    log.info(f"vac/{vector_name} got data: {data}")
-
-    config, _ = load_config("config/llm_config.yaml")
-    vac_configs = config.get("vac")
-    if vac_configs:
-        vac_config = vac_configs.get(vector_name)
-        if not vac_config:
-            log.warning("Not a local configured VAC, may be a remote config not synced yet")
-
-    if trace and vac_config:
-        trace.update(input=data, metadata=vac_config)
-
-    user_input = data.pop('user_input').strip()
-    stream_wait_time = data.pop('stream_wait_time', 7)
-    stream_timeout = data.pop('stream_timeout', 120)
-    chat_history = data.pop('chat_history', None)
-    vector_name = data.pop('vector_name', vector_name)
-
-    log.info("Turning chat_history into paired tuples")
-    paired_messages = extract_chat_history(chat_history)
-
-    all_input = {'user_input': user_input, 
-                 'vector_name': vector_name, 
-                 'chat_history': paired_messages, 
-                 'stream_wait_time': stream_wait_time,
-                 'stream_timeout': stream_timeout,
-                 'kwargs': data}
-
-    if trace:
-        span = trace.span(
-            name="VAC",
-            metadata=vac_config,
-            input = all_input
-        )
-    command_response = handle_special_commands(user_input, vector_name, paired_messages)
-    if command_response is not None:
-        if trace:
-            trace.update(output=jsonify(command_response))
-    
-    return {
-        "trace": trace,
-        "span": span,
-        "command_response": command_response,
-        "all_input": all_input,
-        "vac_config": vac_config
-    }
-
-
-def handle_file_upload(file, vector_name):
-    """
-    Handle file upload and store the file in Google Cloud Storage.
-
-    This function saves the uploaded file locally, uploads it to Google Cloud Storage,
-    and then removes the local copy.
-
-    Args:
-        file (FileStorage): The uploaded file.
-        vector_name (str): The name of the vector for the request.
-
-    Returns:
-        tuple: A tuple containing the URI of the uploaded file and its MIME type.
-
-    Raises:
-        Exception: If the file upload fails.
-
-    Example:
-        uri, mime_type = handle_file_upload(file, "example_vector")
-    """
-    try:
-        file.save(file.filename)
-        image_uri = add_file_to_gcs(file.filename, vector_name)
-        os.remove(file.filename)  # Clean up the saved file
-        return image_uri, file.mimetype
-    except Exception as e:
-        raise Exception(f'File upload failed: {str(e)}')