sunholo 0.89.2__py3-none-any.whl → 0.89.4__py3-none-any.whl

sunholo/agents/flask/vac_routes.py

@@ -2,6 +2,7 @@ import json
 import traceback
 import datetime
 import uuid
+import random
 
 from ...agents import extract_chat_history, handle_special_commands
 from ...qna.parsers import parse_output
@@ -18,12 +19,12 @@ from datetime import timedelta
 try:
     from flask import request, jsonify, Response
 except ImportError:
-    pass
+    pass 
 
 try:
-    from langfuse.decorators import langfuse_context, observe
+    from ..pubsub import PubSubManager
 except ImportError:
-    pass    
+    PubSubManager = None
 
 # Cache dictionary to store validated API keys
 api_key_cache = {}
@@ -221,8 +222,38 @@ if __name__ == "__main__":
             generation.end(output=response)
             span.end(output=response)
             trace.update(output=response)
+            self.langfuse_eval_response(trace.id, all_input.get('eval_percent', 0.01))
 
         return response
+    
+    def langfuse_eval_response(trace_id, eval_percent=0.01):
+        """
+        Sends an evaluation message based on a probability defined by eval_percent.
+        
+        Args:
+            eval_percent (float): The probability (0 to 1) of triggering the evaluation.
+            trace_id (str): The trace identifier for the evaluation.
+        
+        Returns:
+            None
+        """
+        if eval_percent > 1 or eval_percent < 0:
+            raise ValueError("eval_percent must be a float between 0 and 1.")
+
+        # Generate a random float between 0 and 1
+        random_value = random.random()
+        
+        # Check if evaluation should be triggered
+        if random_value < eval_percent:
+            if PubSubManager:
+                try:
+                    pubsub_manager = PubSubManager("langfuse_evals", pubsub_topic="topicid-to-langfuse-eval")
+                    the_data = {"trace_id": trace_id}
+                    pubsub_manager.publish_message(the_data)
+                except Exception as e:
+                    log.warning(f"Could not publish message for 'langfuse_evals' to topicid-to-langfuse-eval - {str(e)}")
+        else:
+            log.info(f"Did not do Langfuse eval due to random sampling not passed: {eval_percent=}")
 
     def handle_process_vac(self, vector_name):
         observed_vac_interpreter = self.vac_interpreter
@@ -268,6 +299,7 @@ if __name__ == "__main__":
         if trace:
             span.end(output=jsonify(bot_output))
             trace.update(output=jsonify(bot_output)) 
+            self.langfuse_eval_response(trace.id, all_input.get('eval_percent', 0.01))
 
         # {'answer': 'output'}
         return jsonify(bot_output)

sunholo/langfuse/evals.py

@@ -38,8 +38,9 @@ def pubsub_to_evals(data: dict, eval_funcs: list=[eval_length]) -> dict:
 def direct_langfuse_evals(data, eval_funcs: list=[eval_length]):
     if 'trace_id' not in data:
         raise ValueError('No trace_id found in data')
-    
-    return do_evals(data['trace_id'], eval_funcs, **data)
+    trace_id = data.pop('trace_id', None)
+
+    return do_evals(trace_id, eval_funcs, **data)
 
 
 def do_evals(trace_id, eval_funcs: list=[eval_length], **kwargs) -> dict:
@@ -51,7 +52,7 @@ def do_evals(trace_id, eval_funcs: list=[eval_length], **kwargs) -> dict:
     )
  
     # Fetch the latest trace (or modify as needed to fetch a specific trace)
-    trace = langfuse.fetch_trace(id=trace_id)
+    trace = langfuse.get_trace(id=trace_id)
 
     if trace.output is None:
         raise ValueError("Trace {trace.name} had no generated output, it was skipped")
@@ -64,7 +65,7 @@ def do_evals(trace_id, eval_funcs: list=[eval_length], **kwargs) -> dict:
 
         eval_name = eval_func.__name__
 
-        if 'score' or 'reason' not in eval_result:
+        if 'score' and 'reason' not in eval_result:
             raise ValueError(f"Trace {trace.name} using {eval_name=} did not return a dict with 'score' and 'reason': {eval_result=}")
         
         log.info(f"TraceId {trace.id} with name {trace.name} had {eval_name=} with score {eval_result=}")

{sunholo-0.89.2.dist-info → sunholo-0.89.4.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.89.2
+Version: 0.89.4
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.89.2.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.89.4.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.89.2.dist-info → sunholo-0.89.4.dist-info}/RECORD

@@ -14,7 +14,7 @@ sunholo/agents/fastapi/qna_routes.py,sha256=lKHkXPmwltu9EH3RMwmD153-J6pE7kWQ4BhB
 sunholo/agents/flask/__init__.py,sha256=poJDKMr2qj8qMb99JqCvCPSiEt1tj2tLQ3hKW3f2aVw,107
 sunholo/agents/flask/base.py,sha256=FgSaCODyoTtlstJtsqlLPScdgRUtv9_plxftdzHdVFo,809
 sunholo/agents/flask/qna_routes.py,sha256=uwUD1yrzOPH27m2AXpiQrPk_2VfJOQOM6dAynOWQtoQ,22532
-sunholo/agents/flask/vac_routes.py,sha256=e8miEz997ELiMJlOdTZesi40E0E2sNA_lz3cEU77e2M,21094
+sunholo/agents/flask/vac_routes.py,sha256=U5JH1mj-3i2BbIVirnyUs_0s5oZ0tX1IWGkV_g7WW5k,22538
 sunholo/archive/__init__.py,sha256=qNHWm5rGPVOlxZBZCpA1wTYPbalizRT7f8X4rs2t290,31
 sunholo/archive/archive.py,sha256=PxVfDtO2_2ZEEbnhXSCbXLdeoHoQVImo4y3Jr2XkCFY,1204
 sunholo/auth/__init__.py,sha256=TeP-OY0XGxYV_8AQcVGoh35bvyWhNUcMRfhuD5l44Sk,91
@@ -94,7 +94,7 @@ sunholo/invoke/direct_vac_func.py,sha256=fuTJlH5PsqWhN_yVMaWisHCTZU1JEUz8I8yVbWs
 sunholo/invoke/invoke_vac_utils.py,sha256=sJc1edHTHMzMGXjji1N67c3iUaP7BmAL5nj82Qof63M,2053
 sunholo/langfuse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/langfuse/callback.py,sha256=jl0SZsFS53uMW9DGeM9SOL_EsRZsba0wwFGLqKzu9_U,1684
-sunholo/langfuse/evals.py,sha256=jGST6yNrgiH8hbXTGWNpu0dopcWHkYPMf1HLhB40gE4,2880
+sunholo/langfuse/evals.py,sha256=aGrU2DOR2hmI0ST613gBIT0v6IhEF1MIK1aOpuD2yR0,2909
 sunholo/langfuse/prompts.py,sha256=27BsVfihM6-h1jscbkGSO4HsATl-d4ZN6tcNCVztWoY,1300
 sunholo/llamaindex/__init__.py,sha256=DlY_cHWCsVEV1C5WBgDdHRgOMlJc8pDoCRukUJ8PT9w,88
 sunholo/llamaindex/get_files.py,sha256=6rhXCDqQ_lrIapISQ_OYQDjiSATXvS_9m3qq53-oIl0,781
@@ -144,9 +144,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=q_phxgGX2TG2j2MXNULF2xGzQnQPENwjPN9nZ_A9Gh0,7526
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.89.2.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.89.2.dist-info/METADATA,sha256=aXFfl6kKHO1A9B6-PtLMl7TKme8xRT5S57ZJQ4Euac4,7706
-sunholo-0.89.2.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
-sunholo-0.89.2.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.89.2.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.89.2.dist-info/RECORD,,
+sunholo-0.89.4.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.89.4.dist-info/METADATA,sha256=vp8lN-BGsMZzN8sUmHdoQuctkysZB_Iv52vLacNaAZg,7706
+sunholo-0.89.4.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
+sunholo-0.89.4.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.89.4.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.89.4.dist-info/RECORD,,