sunholo 0.89.1__tar.gz → 0.89.4__tar.gz

{sunholo-0.89.1 → sunholo-0.89.4}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.89.1
+Version: 0.89.4
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.89.1.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.89.4.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.89.1 → sunholo-0.89.4}/setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-version = '0.89.1'
+version = '0.89.4'
 
 setup(
     name='sunholo',

{sunholo-0.89.1 → sunholo-0.89.4}/sunholo/agents/dispatch_to_qa.py

@@ -20,6 +20,12 @@ from .langserve import prepare_request_data
 
 from .route import route_endpoint
 
+try:
+    from langfuse import Langfuse
+    langfuse = Langfuse()
+except ImportError:
+    langfuse = None
+
 def prep_request_payload(user_input, chat_history, vector_name, stream, **kwargs):
     """
     Prepares the request payload for sending a query to the QA system.
@@ -79,9 +85,21 @@ def prep_request_payload(user_input, chat_history, vector_name, stream, **kwargs
 
         if 'vector_name' not in qna_data:
             qna_data['vector_name'] = vector_name
+    
+    qna_data['trace_id'] = add_langfuse_trace(qna_endpoint)
 
     return qna_endpoint, qna_data
 
+def add_langfuse_trace(qna_endpoint):
+    if not langfuse:
+        return None
+    
+    trace = langfuse.trace(name = f'auto/{qna_endpoint}')
+
+    log.info('Adding langfuse trace {trace.id}')
+
+    return trace.id
+
 def send_to_qa(user_input, vector_name, chat_history, stream=False, **kwargs):
     """
     Sends a query to the QA system synchronously.

{sunholo-0.89.1 → sunholo-0.89.4}/sunholo/agents/flask/vac_routes.py

@@ -2,6 +2,7 @@ import json
 import traceback
 import datetime
 import uuid
+import random
 
 from ...agents import extract_chat_history, handle_special_commands
 from ...qna.parsers import parse_output
@@ -18,12 +19,12 @@ from datetime import timedelta
 try:
     from flask import request, jsonify, Response
 except ImportError:
-    pass
+    pass 
 
 try:
-    from langfuse.decorators import langfuse_context, observe
+    from ..pubsub import PubSubManager
 except ImportError:
-    pass    
+    PubSubManager = None
 
 # Cache dictionary to store validated API keys
 api_key_cache = {}
@@ -161,7 +162,7 @@ if __name__ == "__main__":
 
 
     def handle_stream_vac(self, vector_name):
-        observed_stream_interpreter = observe()(self.stream_interpreter)
+        observed_stream_interpreter = self.stream_interpreter
         prep = self.prep_vac(request, vector_name)
         log.info(f"Processing prep: {prep}")
         trace = prep["trace"]
@@ -221,11 +222,41 @@ if __name__ == "__main__":
             generation.end(output=response)
             span.end(output=response)
             trace.update(output=response)
+            self.langfuse_eval_response(trace.id, all_input.get('eval_percent', 0.01))
 
         return response
+    
+    def langfuse_eval_response(trace_id, eval_percent=0.01):
+        """
+        Sends an evaluation message based on a probability defined by eval_percent.
+        
+        Args:
+            eval_percent (float): The probability (0 to 1) of triggering the evaluation.
+            trace_id (str): The trace identifier for the evaluation.
+        
+        Returns:
+            None
+        """
+        if eval_percent > 1 or eval_percent < 0:
+            raise ValueError("eval_percent must be a float between 0 and 1.")
+
+        # Generate a random float between 0 and 1
+        random_value = random.random()
+        
+        # Check if evaluation should be triggered
+        if random_value < eval_percent:
+            if PubSubManager:
+                try:
+                    pubsub_manager = PubSubManager("langfuse_evals", pubsub_topic="topicid-to-langfuse-eval")
+                    the_data = {"trace_id": trace_id}
+                    pubsub_manager.publish_message(the_data)
+                except Exception as e:
+                    log.warning(f"Could not publish message for 'langfuse_evals' to topicid-to-langfuse-eval - {str(e)}")
+        else:
+            log.info(f"Did not do Langfuse eval due to random sampling not passed: {eval_percent=}")
 
     def handle_process_vac(self, vector_name):
-        observed_vac_interpreter = observe()(self.vac_interpreter)
+        observed_vac_interpreter = self.vac_interpreter
         prep = self.prep_vac(request, vector_name)
         log.debug(f"Processing prep: {prep}")
         trace = prep["trace"]
@@ -268,6 +299,7 @@ if __name__ == "__main__":
         if trace:
             span.end(output=jsonify(bot_output))
             trace.update(output=jsonify(bot_output)) 
+            self.langfuse_eval_response(trace.id, all_input.get('eval_percent', 0.01))
 
         # {'answer': 'output'}
         return jsonify(bot_output)
@@ -365,7 +397,7 @@ if __name__ == "__main__":
             "kwargs": data
         }
 
-        observed_stream_interpreter = observe()(self.stream_interpreter)
+        observed_stream_interpreter = self.stream_interpreter
 
         response_id = str(uuid.uuid4())
         
@@ -416,7 +448,7 @@ if __name__ == "__main__":
             return Response(generate_response_content(), content_type='text/plain; charset=utf-8')
 
         try:
-            observed_vac_interpreter = observe()(self.vac_interpreter)
+            observed_vac_interpreter = self.vac_interpreter
             bot_output = observed_vac_interpreter(
                 question=user_message,
                 vector_name=vector_name,
@@ -437,7 +469,7 @@ if __name__ == "__main__":
             return self.make_openai_response(user_message, vector_name, f'ERROR: {str(err)}')
 
 
-    def create_langfuse_trace(self, request, vector_name):
+    def create_langfuse_trace(self, request, vector_name, trace_id):
         try:
             from langfuse import Langfuse
             langfuse = Langfuse()
@@ -451,11 +483,12 @@ if __name__ == "__main__":
         message_source = request.headers.get("X-Message-Source")
 
         package_version = sunholo_version()
-        tags = [package_version]
+        tags = [package_version, "autogenerated"]
         if message_source:
             tags.append(message_source)
 
         return langfuse.trace(
+            id=trace_id, 
             name = f"/vac/{vector_name}",
             user_id = user_id,
             session_id = session_id,
@@ -464,8 +497,6 @@ if __name__ == "__main__":
         )
 
     def prep_vac(self, request, vector_name):
-        trace = self.create_langfuse_trace(request, vector_name)
-        span = None
 
         if request.content_type.startswith('application/json'):
             data = request.get_json()
@@ -490,6 +521,10 @@ if __name__ == "__main__":
 
         log.info(f"vac/{vector_name} got data: {data}")
 
+        trace_id = data.get('trace_id')
+        trace = self.create_langfuse_trace(request, vector_name, trace_id)
+        span = None
+
         config, _ = load_config("config/llm_config.yaml")
         vac_configs = config.get("vac")
         if vac_configs:

{sunholo-0.89.1 → sunholo-0.89.4}/sunholo/langfuse/evals.py

@@ -29,18 +29,30 @@ def pubsub_to_evals(data: dict, eval_funcs: list=[eval_length]) -> dict:
 
     if 'trace_id' not in message_data:
         raise ValueError('No trace_id found in message data')
-    
+
+    trace_id = message_data.pop('trace_id', None)
+
+    return do_evals(trace_id, eval_funcs, **message_data)
+
+
+def direct_langfuse_evals(data, eval_funcs: list=[eval_length]):
+    if 'trace_id' not in data:
+        raise ValueError('No trace_id found in data')
+    trace_id = data.pop('trace_id', None)
+
+    return do_evals(trace_id, eval_funcs, **data)
+
+
+def do_evals(trace_id, eval_funcs: list=[eval_length], **kwargs) -> dict:
     # Initialize Langfuse with environment variables
     langfuse = Langfuse(
         secret_key=os.environ["LANGFUSE_SECRET_KEY"],
         public_key=os.environ["LANGFUSE_PUBLIC_KEY"],
         host=os.environ["LANGFUSE_HOST"]
     )
-
-    trace_id = message_data.pop('trace_id', None)
  
     # Fetch the latest trace (or modify as needed to fetch a specific trace)
-    trace = langfuse.fetch_trace(id=trace_id)
+    trace = langfuse.get_trace(id=trace_id)
 
     if trace.output is None:
         raise ValueError("Trace {trace.name} had no generated output, it was skipped")
@@ -53,7 +65,7 @@ def pubsub_to_evals(data: dict, eval_funcs: list=[eval_length]) -> dict:
 
         eval_name = eval_func.__name__
 
-        if 'score' or 'reason' not in eval_result:
+        if 'score' and 'reason' not in eval_result:
             raise ValueError(f"Trace {trace.name} using {eval_name=} did not return a dict with 'score' and 'reason': {eval_result=}")
         
         log.info(f"TraceId {trace.id} with name {trace.name} had {eval_name=} with score {eval_result=}")
@@ -64,7 +76,7 @@ def pubsub_to_evals(data: dict, eval_funcs: list=[eval_length]) -> dict:
             name=eval_name,  # Use the function name as the evaluation name
             value=eval_result["score"],
             comment=eval_result["reason"],
-            **message_data
+            **kwargs
         )
     
     return {"trace_id": trace.id, "eval_results": eval_results}

{sunholo-0.89.1 → sunholo-0.89.4}/sunholo.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.89.1
+Version: 0.89.4
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.89.1.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.89.4.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0