sunholo 0.139.1__py3-none-any.whl → 0.140.4__py3-none-any.whl

sunholo/agents/__init__.py

@@ -2,6 +2,6 @@ from .chat_history import extract_chat_history
 from .dispatch_to_qa import send_to_qa, send_to_qa_async
 from .pubsub import process_pubsub
 from .special_commands import handle_special_commands, app_to_store, handle_files
-from .flask import register_qna_routes, create_app, VACRoutes
+from .flask import create_app, VACRoutes
 from .fastapi import register_qna_fastapi_routes, create_fastapi_app
 from .swagger import config_to_swagger

sunholo/agents/chat_history.py

@@ -1,6 +1,69 @@
 import json
 from ..custom_logging import log
 
+
+async def extract_chat_history_async(chat_history=None):
+    """
+    Extracts paired chat history between human and AI messages.
+    
+    For this lightweight processing, we use a simpler approach that minimizes overhead.
+    
+    Args:
+        chat_history (list): List of chat messages.
+    
+    Returns:
+        list: List of tuples with paired human and AI messages.
+    """
+    if not chat_history:
+        log.info("No chat history found")
+        return []
+
+    log.info(f"Extracting chat history: {chat_history}")
+    paired_messages = []
+    
+    # Handle special case of initial bot message
+    if chat_history and is_bot(chat_history[0]):
+        first_message = chat_history[0]
+        log.info(f"Extracting first_message: {first_message}")
+        blank_human_message = {"name": "Human", "content": "", "embeds": []}
+        
+        # Since create_message_element is so lightweight, we don't need async here
+        blank_element = create_message_element(blank_human_message)
+        bot_element = create_message_element(first_message)
+        
+        paired_messages.append((blank_element, bot_element))
+        chat_history = chat_history[1:]
+    
+    # Pre-process all messages in one batch (more efficient than one-by-one)
+    message_types = []
+    message_contents = []
+    
+    for message in chat_history:
+        is_human_msg = is_human(message)
+        is_bot_msg = is_bot(message)
+        
+        # Extract content for all messages at once
+        content = create_message_element(message)
+        
+        message_types.append((is_human_msg, is_bot_msg))
+        message_contents.append(content)
+    
+    # Pair messages efficiently
+    last_human_message = ""
+    for i, ((is_human_msg, is_bot_msg), content) in enumerate(zip(message_types, message_contents)):
+        if is_human_msg:
+            last_human_message = content
+            log.info(f"Extracted human message: {last_human_message}")
+        elif is_bot_msg:
+            ai_message = content
+            log.info(f"Extracted AI message: {ai_message}")
+            paired_messages.append((last_human_message, ai_message))
+            last_human_message = ""
+    
+    log.info(f"Paired messages: {paired_messages}")
+    return paired_messages
+
+
 def extract_chat_history(chat_history=None):
     """
     Extracts paired chat history between human and AI messages.

sunholo/agents/flask/__init__.py

@@ -1,3 +1,2 @@
-from .qna_routes import register_qna_routes
 from .base import create_app
 from .vac_routes import VACRoutes

sunholo/agents/flask/vac_routes.py

@@ -8,6 +8,7 @@ import inspect
 import asyncio
 
 from ...agents import extract_chat_history, handle_special_commands
+from ..chat_history import extract_chat_history_async
 from ...qna.parsers import parse_output
 from ...streaming import start_streaming_chat, start_streaming_chat_async
 from ...archive import archive_qa
@@ -57,11 +58,12 @@ if __name__ == "__main__":
 ```
     
     """
-    def __init__(self, app, stream_interpreter, vac_interpreter=None, additional_routes=None):
+    def __init__(self, app, stream_interpreter: callable, vac_interpreter:callable=None, additional_routes:dict=None, async_stream:bool=False):
         self.app = app
         self.stream_interpreter = stream_interpreter
         self.vac_interpreter = vac_interpreter or partial(self.vac_interpreter_default)
         self.additional_routes = additional_routes if additional_routes is not None else []
+        self.async_stream = async_stream
         self.register_routes()
         
 
@@ -94,12 +96,16 @@ if __name__ == "__main__":
         # Basic routes
         self.app.route("/", methods=['GET'])(self.home)
         self.app.route("/health", methods=['GET'])(self.health)
-
-        # Streaming VAC
-        self.app.route('/vac/streaming/<vector_name>', 
-                       methods=['POST'], 
-                       provide_automatic_options=False)(self.handle_stream_vac)
-
+        
+        if self.async_stream:  # Use async treatment
+            log.info("async_stream enabled")
+            self.app.route('/vac/streaming/<vector_name>', 
+                        methods=['POST'], 
+                        provide_automatic_options=False)(self.handle_stream_vac_async)
+        else:
+            self.app.route('/vac/streaming/<vector_name>', 
+                        methods=['POST'], 
+                        provide_automatic_options=False)(self.handle_stream_vac)
         # Static VAC
         self.app.route('/vac/<vector_name>', 
                        methods=['POST'], 
@@ -332,6 +338,51 @@ if __name__ == "__main__":
 
         return response
 
+    async def handle_stream_vac_async(self, vector_name):
+        observed_stream_interpreter = self.stream_interpreter
+        is_async = inspect.iscoroutinefunction(self.stream_interpreter)
+
+        if not is_async:
+            raise ValueError(f"Stream interpreter must be async: {observed_stream_interpreter}")
+
+        # Use the async version of prep_vac
+        prep = await self.prep_vac_async(request, vector_name)
+        log.info(f"Processing async prep: {prep}")
+        all_input = prep["all_input"]
+
+        log.info(f'Streaming async data with: {all_input}')
+
+        async def generate_response_content():
+            try:
+                # Direct async handling without the queue/thread approach
+                async_gen = start_streaming_chat_async(
+                    question=all_input["user_input"],
+                    vector_name=vector_name,
+                    qna_func_async=observed_stream_interpreter,
+                    chat_history=all_input["chat_history"],
+                    wait_time=all_input["stream_wait_time"],
+                    timeout=all_input["stream_timeout"],
+                    **all_input["kwargs"]
+                )
+                
+                log.info(f"{async_gen=}")
+                async for chunk in async_gen:
+                    if isinstance(chunk, dict) and 'answer' in chunk:
+                        await archive_qa(chunk, vector_name)
+                        yield json.dumps(chunk)
+                    else:
+                        yield chunk
+
+            except Exception as e:
+                yield f"Streaming async Error: {str(e)} {traceback.format_exc()}"
+
+        response = Response(generate_response_content(), content_type='text/plain; charset=utf-8')
+        response.headers['Transfer-Encoding'] = 'chunked'
+
+        log.debug(f"streaming async response: {response}")
+
+        return response
+
     @staticmethod
     async def _async_generator_to_stream(async_gen_func):
         """Helper function to stream the async generator's values to the client."""
@@ -699,6 +750,66 @@ if __name__ == "__main__":
             "vac_config": vac_config
         }
 
+    async def prep_vac_async(self, request, vector_name):
+        """Async version of prep_vac."""
+        # Parse request data
+        if request.content_type.startswith('application/json'):
+            data = request.get_json()
+        elif request.content_type.startswith('multipart/form-data'):
+            data = request.form.to_dict()
+            if 'file' in request.files:
+                file = request.files['file']
+                if file.filename != '':
+                    log.info(f"Found file: {file.filename} to upload to GCS")
+                    try:
+                        # Make file upload async if possible
+                        image_uri, mime_type = await self.handle_file_upload_async(file, vector_name)
+                        data["image_uri"] = image_uri
+                        data["mime"] = mime_type
+                    except Exception as e:
+                        log.error(traceback.format_exc())
+                        return jsonify({'error': str(e), 'traceback': traceback.format_exc()}), 500
+                else:
+                    log.error("No file selected")
+                    return jsonify({"error": "No file selected"}), 400
+        else:
+            return jsonify({"error": "Unsupported content type"}), 400
+
+        log.info(f"vac/{vector_name} got data: {data}")
+
+        # Run these operations concurrently
+        tasks = []
+        
+        # Extract other data while configs load
+        user_input = data.pop('user_input').strip()
+        stream_wait_time = data.pop('stream_wait_time', 7)
+        stream_timeout = data.pop('stream_timeout', 120)
+        chat_history = data.pop('chat_history', None)
+        vector_name_param = data.pop('vector_name', vector_name)
+        data.pop('trace_id', None)  # to ensure not in kwargs
+        
+        # Task 3: Process chat history
+        chat_history_task = asyncio.create_task(extract_chat_history_async(chat_history))
+        tasks.append(chat_history_task)
+        
+        # Await all tasks concurrently
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        paired_messages = results[0] if not isinstance(results[0], Exception) else []
+        
+        # Only create span after we have trace
+        all_input = {
+            'user_input': user_input, 
+            'vector_name': vector_name_param, 
+            'chat_history': paired_messages, 
+            'stream_wait_time': stream_wait_time,
+            'stream_timeout': stream_timeout,
+            'kwargs': data
+        }
+        
+        return {
+            "all_input": all_input
+        }
 
     def handle_file_upload(self, file, vector_name):
         try:

{sunholo-0.139.1.dist-info → sunholo-0.140.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.139.1
+Version: 0.140.4
 Summary: AI DevOps - a package to help deploy GenAI to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.139.1.dist-info → sunholo-0.140.4.dist-info}/RECORD

@@ -1,8 +1,8 @@
 sunholo/__init__.py,sha256=InRbX4V0-qdNHo9zYH3GEye7ASLR6LX8-SMvPV4Jsaw,1212
 sunholo/custom_logging.py,sha256=JXZTnXp_DixP3jwYfKw4LYRDS9IuTq7ctCgfZbI2rxA,22023
 sunholo/langchain_types.py,sha256=uZ4zvgej_f7pLqjtu4YP7qMC_eZD5ym_5x4pyvA1Ih4,1834
-sunholo/agents/__init__.py,sha256=X2I3pPkGeKWjc3d0QgSpkTyqD8J8JtrEWqwrumf1MMc,391
-sunholo/agents/chat_history.py,sha256=Gph_CdlP2otYnNdR1q1Umyyyvcad2F6K3LxU5yBQ9l0,5387
+sunholo/agents/__init__.py,sha256=AauG3l0y4r5Fzx1zJfZ634M4o-0o7B7J5T8k_gPvNqE,370
+sunholo/agents/chat_history.py,sha256=4jGCHBP8dZfUjSJPxgKyh6nOqhnHRn1x9U3CnGb0I5E,7624
 sunholo/agents/dispatch_to_qa.py,sha256=NHihwAoCJ5_Lk11e_jZnucVUGQyZHCB-YpkfMHBCpQk,8882
 sunholo/agents/langserve.py,sha256=C46ph2mnygr6bdHijYWYyfQDI9ylAF0_9Kx2PfcCJpU,4414
 sunholo/agents/pubsub.py,sha256=TscZN_6am6DfaQkC-Yl18ZIBOoLE-0nDSiil6GpQEh4,1344
@@ -12,10 +12,9 @@ sunholo/agents/swagger.py,sha256=2tzGmpveUMmTREykZvVnDj3j295wyOMu7mUFDnXdY3c,106
 sunholo/agents/fastapi/__init__.py,sha256=S_pj4_bTUmDGoq_exaREHlOKThi0zTuGT0VZY0YfODQ,88
 sunholo/agents/fastapi/base.py,sha256=W-cyF8ZDUH40rc-c-Apw3-_8IIi2e4Y9qRtnoVnsc1Q,2521
 sunholo/agents/fastapi/qna_routes.py,sha256=lKHkXPmwltu9EH3RMwmD153-J6pE7kWQ4BhBlV3to-s,3864
-sunholo/agents/flask/__init__.py,sha256=poJDKMr2qj8qMb99JqCvCPSiEt1tj2tLQ3hKW3f2aVw,107
+sunholo/agents/flask/__init__.py,sha256=dEoByI3gDNUOjpX1uVKP7uPjhfFHJubbiaAv3xLopnk,63
 sunholo/agents/flask/base.py,sha256=HLz3Z5efWaewTwSFEM6JH48NA9otoJBoVFJlARGk9L8,788
-sunholo/agents/flask/qna_routes.py,sha256=uwUD1yrzOPH27m2AXpiQrPk_2VfJOQOM6dAynOWQtoQ,22532
-sunholo/agents/flask/vac_routes.py,sha256=9bytTeoOJQOYxsPGLIXLItDmnbB9zDXmYM0lBIwDe8w,28335
+sunholo/agents/flask/vac_routes.py,sha256=Dk9QrPvXNRzAWxaTWsYgHVxmK-Rjrvgd6-sAuvqt9P8,33236
 sunholo/archive/__init__.py,sha256=qNHWm5rGPVOlxZBZCpA1wTYPbalizRT7f8X4rs2t290,31
 sunholo/archive/archive.py,sha256=PxVfDtO2_2ZEEbnhXSCbXLdeoHoQVImo4y3Jr2XkCFY,1204
 sunholo/auth/__init__.py,sha256=TeP-OY0XGxYV_8AQcVGoh35bvyWhNUcMRfhuD5l44Sk,91
@@ -169,9 +168,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.139.1.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.139.1.dist-info/METADATA,sha256=vOG7X6ZpBgF3og9_BNDil-Loy2tAW38orcqYo3ObTTk,10067
-sunholo-0.139.1.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-sunholo-0.139.1.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.139.1.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.139.1.dist-info/RECORD,,
+sunholo-0.140.4.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.140.4.dist-info/METADATA,sha256=oKjtRKqFPtwaoV177G0nRWfv3P9xfGB3U4fonaffJrk,10067
+sunholo-0.140.4.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+sunholo-0.140.4.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.140.4.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.140.4.dist-info/RECORD,,

{sunholo-0.139.1.dist-info → sunholo-0.140.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.4.0)
+Generator: setuptools (80.7.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

sunholo/agents/flask/qna_routes.py

@@ -1,604 +0,0 @@
-#   Copyright [2024] [Holosun ApS]
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-
-import json
-import traceback
-import uuid
-
-from ...agents import extract_chat_history, handle_special_commands
-from ...qna.parsers import parse_output
-from ...streaming import start_streaming_chat
-from ...archive import archive_qa
-from ...custom_logging import log
-from ...utils.config import load_config
-from ...utils import ConfigManager
-from ...utils.version import sunholo_version
-import os
-from ...gcs.add_file import add_file_to_gcs, handle_base64_image
-from ..swagger import validate_api_key
-from datetime import datetime, timedelta
-
-try:
-    from flask import request, jsonify, Response
-except ImportError:
-    pass
-
-try:
-    from langfuse.decorators import langfuse_context, observe
-except ImportError:
-    pass    
-
-# Cache dictionary to store validated API keys
-api_key_cache = {}
-cache_duration = timedelta(minutes=5)  # Cache duration
-
-def make_openai_response(user_message, vector_name, answer):
-    response_id = str(uuid.uuid4())
-    log.info(f"openai response: Q: {user_message} to VECTOR_NAME: {vector_name} - A: {answer}")
-    openai_response = {
-        "id": response_id,
-        "object": "chat.completion",
-        "created": str(int(datetime.now().timestamp())),
-        "model": vector_name,
-        "system_fingerprint": sunholo_version(),
-        "choices": [{
-            "index": 0,
-            "message": {
-                "role": "assistant",
-                "content": answer,
-            },
-            "logprobs": None,
-            "finish_reason": "stop"
-        }],
-        "usage": {
-            "prompt_tokens": 0,
-            "completion_tokens": 0,
-            "total_tokens": 0
-        }
-    }
-
-    log.info(f"OpenAI response: {openai_response}")
-    return jsonify(openai_response)
-
-def register_qna_routes(app, stream_interpreter, vac_interpreter):
-    """
-    Register Q&A routes for a Flask application.
-
-    This function sets up multiple routes for handling Q&A operations,
-    including streaming responses and processing static responses.
-
-    Args:
-        app (Flask): The Flask application instance.
-        stream_interpreter (function): Function to handle streaming Q&A responses.
-        vac_interpreter (function): Function to handle static Q&A responses.
-
-    Returns:
-        None
-
-    Example:
-        from flask import Flask
-        app = Flask(__name__)
-
-        def dummy_stream_interpreter(...):
-            ...
-
-        def dummy_vac_interpreter(...):
-            ...
-
-        register_qna_routes(app, dummy_stream_interpreter, dummy_vac_interpreter)
-    """
-    @app.route("/")
-    def home():
-        return jsonify("OK")
-
-    @app.route("/health")
-    def health():
-        return jsonify({"status": "healthy"})
-
-    @app.route('/vac/streaming/<vector_name>', methods=['POST'])
-    def stream_qa(vector_name):
-        """
-        Handle streaming Q&A responses.
-
-        This function sets up a route to handle streaming Q&A responses based on
-        the provided vector name.
-
-        Args:
-            vector_name (str): The name of the vector for the request.
-
-        Returns:
-            Response: A Flask response object streaming the Q&A response content.
-
-        Example:
-            response = stream_qa("example_vector")
-        """
-        observed_stream_interpreter = observe()(stream_interpreter)
-        prep = prep_vac(request, vector_name)
-        log.debug(f"Processing prep: {prep}")
-        trace = prep["trace"]
-        span = prep["span"]
-        command_response = prep["command_response"]
-        vac_config = prep["vac_config"]
-        all_input = prep["all_input"]
-
-        if command_response:
-            return jsonify(command_response)
-
-        log.info(f'Streaming data with: {all_input}')
-        if span:
-            generation = span.generation(
-                name="start_streaming_chat",
-                metadata=vac_config,
-                input = all_input,
-                completion_start_time=datetime.now(),
-                model=vac_config.get("model") or vac_config.get("llm")
-            )
-
-        def generate_response_content():
-
-            for chunk in start_streaming_chat(question=all_input["user_input"],
-                                              vector_name=vector_name,
-                                              qna_func=observed_stream_interpreter,
-                                              chat_history=all_input["chat_history"],
-                                              wait_time=all_input["stream_wait_time"],
-                                              timeout=all_input["stream_timeout"],
-                                              #kwargs
-                                              **all_input["kwargs"]
-                                              ):
-                if isinstance(chunk, dict) and 'answer' in chunk:
-                    # When we encounter the dictionary, we yield it as a JSON string
-                    # and stop the generator.
-                    if trace:
-                        chunk["trace"] = trace.id
-                        chunk["trace_url"] = trace.get_trace_url()
-                    archive_qa(chunk, vector_name)
-                    if trace:
-                        generation.end(output=json.dumps(chunk))
-                        span.end(output=json.dumps(chunk))
-                        trace.update(output=json.dumps(chunk))
-  
-                    return json.dumps(chunk)
-                
-                else:
-                    # Otherwise, we yield the plain text chunks as they come in.
-                    yield chunk
-            
-        # Here, the generator function will handle streaming the content to the client.
-        response = Response(generate_response_content(), content_type='text/plain; charset=utf-8')
-        response.headers['Transfer-Encoding'] = 'chunked'  
-
-        log.debug(f"streaming response: {response}")
-        if trace:
-            generation.end(output=response)
-            span.end(output=response)
-            trace.update(output=response)
-        
-        #if 'user_id' in all_input["kwargs"]:
-        #    kwargs = all_input["kwargs"]
-        #    config = ConfigManager(vector_name)
-        #    add_user_history_rag(kwargs.pop('user_id'), 
-        #                         config, 
-        #                         question=all_input.pop("user_input"), 
-        #                         answer=response.get('answer'), 
-        #                         metadata=all_input)
-
-        return response
-
-    @app.route('/vac/<vector_name>', methods=['POST'])
-    def process_qna(vector_name):
-        """
-        Handle static Q&A responses.
-
-        This function sets up a route to handle static Q&A responses based on
-        the provided vector name.
-
-        Args:
-            vector_name (str): The name of the vector for the request.
-
-        Returns:
-            Response: A Flask response object with the Q&A response content.
-
-        Example:
-            response = process_qna("example_vector")
-        """
-        observed_vac_interpreter = observe()(vac_interpreter)
-        prep = prep_vac(request, vector_name)
-        log.debug(f"Processing prep: {prep}")
-        trace = prep["trace"]
-        span = prep["span"]
-        command_response = prep["command_response"]
-        vac_config = prep["vac_config"]
-        all_input = prep["all_input"]
-
-        if command_response:
-            return jsonify(command_response)
-
-        try:
-            if span:
-                generation = span.generation(
-                    name="vac_interpreter",
-                    metadata=vac_config,
-                    input = all_input,
-                    model=vac_config.get("model") or vac_config.get("llm")
-                )
-            bot_output = observed_vac_interpreter(
-                question=all_input["user_input"],
-                vector_name=vector_name,
-                chat_history=all_input["chat_history"],
-                **all_input["kwargs"]
-            )
-            if span:
-                generation.end(output=bot_output)
-            # {"answer": "The answer", "source_documents": [{"page_content": "The page content", "metadata": "The metadata"}]}
-            bot_output = parse_output(bot_output)
-            if trace:
-                bot_output["trace"] = trace.id
-                bot_output["trace_url"] = trace.get_trace_url()
-            archive_qa(bot_output, vector_name)
-            log.info(f'==LLM Q:{all_input["user_input"]} - A:{bot_output}')
-
-
-        except Exception as err: 
-            bot_output = {'answer': f'QNA_ERROR: An error occurred while processing /vac/{vector_name}: {str(err)} traceback: {traceback.format_exc()}'}
-        
-        if trace:
-            span.end(output=jsonify(bot_output))
-            trace.update(output=jsonify(bot_output)) 
-
-        # {'answer': 'output'}
-        return jsonify(bot_output)
-
-    @app.before_request
-    def check_authentication_header():
-        if request.path.startswith('/openai/'):
-            log.debug(f'Request headers: {request.headers}')
-            # the header forwarded
-            auth_header = request.headers.get('X-Forwarded-Authorization')
-            if auth_header:
-                
-                if auth_header.startswith('Bearer '):
-                    api_key = auth_header.split(' ')[1]  # Assuming "Bearer <api_key>"
-                else:
-                    return jsonify({'error': 'Invalid authorization header does not start with "Bearer " - got: {auth_header}'}), 401
-                
-                endpoints_host = os.getenv('_ENDPOINTS_HOST')
-                if not endpoints_host:
-                    return jsonify({'error': '_ENDPOINTS_HOST environment variable not found'}), 401
-                
-                # Check cache first
-                current_time = datetime.now()
-                if api_key in api_key_cache:
-                    cached_result, cache_time = api_key_cache[api_key]
-                    if current_time - cache_time < cache_duration:
-                        if not cached_result:
-                            return jsonify({'error': 'Invalid cached API key'}), 401
-                        else:
-                            return  # Valid API key, continue to the endpoint
-                    else:
-                        # Cache expired, remove from cache
-                        del api_key_cache[api_key]
-                
-                # Validate API key
-                is_valid = validate_api_key(api_key, endpoints_host)
-                # Update cache
-                api_key_cache[api_key] = (is_valid, current_time)
-                
-                if not is_valid:
-                    return jsonify({'error': 'Invalid API key'}), 401
-            else:
-                return jsonify({'error': 'Missing Authorization header'}), 401
-
-    @app.route('/openai/health', methods=['GET', 'POST'])
-    def openai_health_endpoint():
-        return jsonify({'message': 'Success'})
-
-    @app.route('/openai/v1/chat/completions', methods=['POST'])
-    @app.route('/openai/v1/chat/completions/<vector_name>', methods=['POST'])
-    def openai_compatible_endpoint(vector_name=None):
-        """
-        Handle OpenAI-compatible chat completions.
-
-        This function sets up routes to handle OpenAI-compatible chat completion requests,
-        both with and without a specified vector name.
-
-        Args:
-            vector_name (str, optional): The name of the vector for the request. Defaults to None.
-
-        Returns:
-            Response: A Flask response object with the chat completion content.
-
-        Example:
-            response = openai_compatible_endpoint("example_vector")
-        """
-        data = request.get_json()
-        log.info(f'openai_compatible_endpoint got data: {data} for vector: {vector_name}')
-
-        vector_name = vector_name or data.pop('model', None)
-        messages = data.pop('messages', None)
-        chat_history = data.pop('chat_history', None)
-        stream = data.pop('stream', False)
-
-        if not messages:
-            return jsonify({"error": "No messages provided"}), 400
-
-        user_message = None
-        image_uri = None
-        mime_type = None
-
-        for msg in reversed(messages):
-            if msg['role'] == 'user':
-                if isinstance(msg['content'], list):
-                    for content_item in msg['content']:
-                        if content_item['type'] == 'text':
-                            user_message = content_item['text']
-                        elif content_item['type'] == 'image_url':
-                            base64_data = content_item['image_url']['url']
-                            image_uri, mime_type = handle_base64_image(base64_data, vector_name)
-                else:
-                    user_message = msg['content']
-                break
-
-        if not user_message:
-            return jsonify({"error": "No user message provided"}), 400
-        else:
-            log.info(f"User message: {user_message}")
-        
-        paired_messages = extract_chat_history(chat_history)
-        command_response = handle_special_commands(user_message, vector_name, paired_messages)
-
-        if command_response is not None:
-
-            return make_openai_response(user_message, vector_name, command_response)
-        
-        if image_uri:
-            data["image_uri"] = image_uri
-            data["mime"] = mime_type
-
-        all_input = {
-            "user_input": user_message,
-            "chat_history": chat_history,
-            "kwargs": data
-        }
-
-        observed_stream_interpreter = observe()(stream_interpreter)
-
-        response_id = str(uuid.uuid4())
-        
-        def generate_response_content():
-            for chunk in start_streaming_chat(question=user_message,
-                                            vector_name=vector_name,
-                                            qna_func=observed_stream_interpreter,
-                                            chat_history=all_input["chat_history"],
-                                            wait_time=all_input.get("stream_wait_time", 1),
-                                            timeout=all_input.get("stream_timeout", 60),
-                                            **all_input["kwargs"]
-                                            ):
-                if isinstance(chunk, dict) and 'answer' in chunk:
-                    openai_chunk = {
-                        "id": response_id,
-                        "object": "chat.completion.chunk",
-                        "created": str(int(datetime.now().timestamp())),
-                        "model": vector_name,
-                        "system_fingerprint": sunholo_version(),
-                        "choices": [{
-                            "index": 0,
-                            "delta": {"content": chunk['answer']},
-                            "logprobs": None,
-                            "finish_reason": None
-                        }]
-                    }
-                    yield json.dumps(openai_chunk) + "\n"
-                else:
-                    log.info(f"Unknown chunk: {chunk}")
-
-            final_chunk = {
-                "id": response_id,
-                "object": "chat.completion.chunk",
-                "created": str(int(datetime.now().timestamp())),
-                "model": vector_name,
-                "system_fingerprint": sunholo_version(),
-                "choices": [{
-                    "index": 0,
-                    "delta": {},
-                    "logprobs": None,
-                    "finish_reason": "stop"
-                }]
-            }
-            yield json.dumps(final_chunk) + "\n"
-
-        if stream:
-            log.info("Streaming openai chunks")
-            return Response(generate_response_content(), content_type='text/plain; charset=utf-8')
-
-        try:
-            observed_vac_interpreter = observe()(vac_interpreter)
-            bot_output = observed_vac_interpreter(
-                question=user_message,
-                vector_name=vector_name,
-                chat_history=all_input["chat_history"],
-                **all_input["kwargs"]
-            )
-            bot_output = parse_output(bot_output)
-
-            log.info(f"Bot output: {bot_output}")
-            if bot_output:
-                return make_openai_response(user_message, vector_name, bot_output.get('answer', ''))
-            else:
-                return make_openai_response(user_message, vector_name, 'ERROR: could not find an answer')
-
-        except Exception as err:
-            log.error(f"OpenAI response error: {str(err)} traceback: {traceback.format_exc()}")
-            
-            return make_openai_response(user_message, vector_name, f'ERROR: {str(err)}')
-
-
-def create_langfuse_trace(request, vector_name):
-    """
-    Create a Langfuse trace for tracking requests.
-
-    This function initializes a Langfuse trace object based on the request headers
-    and vector name.
-
-    Args:
-        request (Request): The Flask request object.
-        vector_name (str): The name of the vector for the request.
-
-    Returns:
-        Langfuse.Trace: The Langfuse trace object.
-
-    Example:
-        trace = create_langfuse_trace(request, "example_vector")
-    """
-    try:
-        from langfuse import Langfuse
-        langfuse = Langfuse()
-    except ImportError as err:
-        print(f"No langfuse installed for agents.flask.register_qna_routes, install via `pip install sunholo[http]` - {str(err)}")
-        
-        return None
-
-    user_id = request.headers.get("X-User-ID")
-    session_id = request.headers.get("X-Session-ID")
-    message_source = request.headers.get("X-Message-Source")
-
-    package_version = sunholo_version()
-    tags = [package_version]
-    if message_source:
-        tags.append(message_source)
-
-    return langfuse.trace(
-        name = f"/vac/{vector_name}",
-        user_id = user_id,
-        session_id = session_id,
-        tags = tags,
-        release = f"sunholo-v{package_version}"
-    )
-
-def prep_vac(request, vector_name):
-    """
-    Prepare the input data for a VAC request.
-
-    This function processes the incoming request data, extracts relevant
-    information, and prepares the data for VAC processing.
-
-    Args:
-        request (Request): The Flask request object.
-        vector_name (str): The name of the vector for the request.
-
-    Returns:
-        dict: A dictionary containing prepared input data and metadata.
-
-    Example:
-        prep_data = prep_vac(request, "example_vector")
-    """
-    #trace = create_langfuse_trace(request, vector_name)
-    trace = None
-    span = None
-
-    if request.content_type.startswith('application/json'):
-        data = request.get_json()
-    elif request.content_type.startswith('multipart/form-data'):
-        data = request.form.to_dict()
-        if 'file' in request.files:
-            file = request.files['file']
-            if file.filename != '':
-                log.info(f"Found file: {file.filename} to upload to GCS")
-                try:
-                    image_uri, mime_type = handle_file_upload(file, vector_name)
-                    data["image_uri"] = image_uri
-                    data["image_url"] = image_uri
-                    data["mime"] = mime_type
-                except Exception as e:
-                    log.error(f"Error uploading file: {str(e)}")
-            else:
-                log.info("No file selected to upload to GCS")
-    else:
-        log.warning(f"Error uploading file: Unsupported content type {request.content_type}")
-
-    log.info(f"vac/{vector_name} got data: {data}")
-
-    config, _ = load_config("config/llm_config.yaml")
-    vac_configs = config.get("vac")
-    if vac_configs:
-        vac_config = vac_configs.get(vector_name)
-        if not vac_config:
-            log.warning("Not a local configured VAC, may be a remote config not synced yet")
-
-    if trace and vac_config:
-        trace.update(input=data, metadata=vac_config)
-
-    user_input = data.pop('user_input').strip()
-    stream_wait_time = data.pop('stream_wait_time', 7)
-    stream_timeout = data.pop('stream_timeout', 120)
-    chat_history = data.pop('chat_history', None)
-    vector_name = data.pop('vector_name', vector_name)
-
-    log.info("Turning chat_history into paired tuples")
-    paired_messages = extract_chat_history(chat_history)
-
-    all_input = {'user_input': user_input, 
-                 'vector_name': vector_name, 
-                 'chat_history': paired_messages, 
-                 'stream_wait_time': stream_wait_time,
-                 'stream_timeout': stream_timeout,
-                 'kwargs': data}
-
-    if trace:
-        span = trace.span(
-            name="VAC",
-            metadata=vac_config,
-            input = all_input
-        )
-    command_response = handle_special_commands(user_input, vector_name, paired_messages)
-    if command_response is not None:
-        if trace:
-            trace.update(output=jsonify(command_response))
-    
-    return {
-        "trace": trace,
-        "span": span,
-        "command_response": command_response,
-        "all_input": all_input,
-        "vac_config": vac_config
-    }
-
-
-def handle_file_upload(file, vector_name):
-    """
-    Handle file upload and store the file in Google Cloud Storage.
-
-    This function saves the uploaded file locally, uploads it to Google Cloud Storage,
-    and then removes the local copy.
-
-    Args:
-        file (FileStorage): The uploaded file.
-        vector_name (str): The name of the vector for the request.
-
-    Returns:
-        tuple: A tuple containing the URI of the uploaded file and its MIME type.
-
-    Raises:
-        Exception: If the file upload fails.
-
-    Example:
-        uri, mime_type = handle_file_upload(file, "example_vector")
-    """
-    try:
-        file.save(file.filename)
-        image_uri = add_file_to_gcs(file.filename, vector_name)
-        os.remove(file.filename)  # Clean up the saved file
-        return image_uri, file.mimetype
-    except Exception as e:
-        raise Exception(f'File upload failed: {str(e)}')