sunholo 0.95.4__py3-none-any.whl → 0.96.0__py3-none-any.whl

sunholo/genai/process_funcs_cls.py

@@ -523,7 +523,7 @@ class GenAIFunctionProcessor:
         
         usage_metadata["functions_called"] = functions_called
 
-        big_text = "\n".join(big_result[loop_return:])
+        big_text = "\n".join(big_result[-loop_return:])
 
         return big_text, usage_metadata
 

sunholo/langfuse/evals.py

@@ -4,6 +4,8 @@ import json
 from ..pubsub import decode_pubsub_message
 from langfuse import Langfuse
 import traceback
+
+
 from ..custom_logging import log
 
 # Example of how eval_funcs might be structured
@@ -60,6 +62,7 @@ def do_evals(trace_id, eval_funcs: list=[eval_length], **kwargs) -> dict:
         host=os.environ["LANGFUSE_HOST"]
     )
  
+    log.info(f"do_evals langfuse with {kwargs=}")
     # Fetch the latest trace (or modify as needed to fetch a specific trace)
     trace = langfuse.get_trace(id=trace_id)
 
@@ -67,31 +70,39 @@ def do_evals(trace_id, eval_funcs: list=[eval_length], **kwargs) -> dict:
         raise ValueError("Trace {trace.name} had no generated output, it was skipped")
 
     # Run the evaluation functions
-    eval_results = []
+    # an eval_response can have multiple eval_results
+    eval_responses = []
     for eval_func in eval_funcs:
         try:
-            eval_result = eval_func(trace)  # Assuming eval_func returns a dict with 'score' and 'reason'
+            eval_response = eval_func(trace, **kwargs)  # Assuming eval_func returns a dict with 'score' and 'reason'
         except Exception as e:
-            eval_result = {"score": 0, "reason":f"ERROR: {str(e)} traceback: {traceback.format_exc()}"}
-        eval_results.append(eval_result)
+            eval_response = {"score": 0, "reason":f"ERROR: {str(e)} traceback: {traceback.format_exc()}"}
+        eval_responses.append(eval_response)
 
         eval_name = eval_func.__name__
 
-        if 'score' and 'reason' not in eval_result:
-            log.error(f"Trace {trace.name} using {eval_name=} did not return a dict with 'score' and 'reason': {eval_result=}")
-            eval_result = {"score": 0, "reason": f"malformed eval_result: {eval_result}"}
-        
-        log.info(f"TraceId {trace.id} with name {trace.name} had {eval_name=} with score {eval_result=}")
+        if isinstance(eval_response, list):
+            eval_results = eval_response
+        else:
+            eval_results = [eval_response]
         
-        # Submit the evaluation to Langfuse
-        langfuse.score(
-            trace_id=trace.id,
-            name=eval_name,  # Use the function name as the evaluation name
-            value=eval_result["score"],
-            comment=eval_result["reason"],
-            **kwargs
-        )
+        for eval_result in eval_results:
+
+            if 'score' and 'reason' not in eval_result:
+                log.error(f"Trace {trace.name} using {eval_name=} did not return a dict with 'score' and 'reason': {eval_result=}")
+                eval_result = {"score": 0, "reason": f"malformed eval_result: {eval_result}"}
+            
+            log.info(f"TraceId {trace.id} with name {trace.name} had {eval_name=} with score {eval_result=}")
+            
+            # Submit the evaluation to Langfuse
+            langfuse.score(
+                trace_id=trace.id,
+                name=eval_name,  # Use the function name as the evaluation name
+                value=eval_result["score"],
+                comment=eval_result["reason"],
+                **kwargs
+            )
     
-    return {"trace_id": trace.id, "eval_results": eval_results}
+    return {"trace_id": trace.id, "eval_results": eval_responses}
 
 

{sunholo-0.95.4.dist-info → sunholo-0.96.0.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.95.4
+Version: 0.96.0
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.95.4.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.96.0.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.95.4.dist-info → sunholo-0.96.0.dist-info}/RECORD

@@ -86,7 +86,7 @@ sunholo/gcs/download_url.py,sha256=q1NiJSvEhdNrmU5ZJ-sBCMC_J5CxzrajY8LRgdPOV_M,6
 sunholo/gcs/metadata.py,sha256=oQLcXi4brsZ74aegWyC1JZmhlaEV270HS5_UWtAYYWE,898
 sunholo/genai/__init__.py,sha256=dBl6IA3-Fx6-Vx81r0XqxHlUq6WeW1iDX188dpChu8s,115
 sunholo/genai/init.py,sha256=yG8E67TduFCTQPELo83OJuWfjwTnGZsyACospahyEaY,687
-sunholo/genai/process_funcs_cls.py,sha256=OCLa_yPjVlE7wY1v54SY0M6b5lpU2l3guRHb-qYOk2A,24359
+sunholo/genai/process_funcs_cls.py,sha256=M6vE_-Mi8v1Od8iny3h9-pRA5KbO5n4hiJKr1yPN9BU,24360
 sunholo/genai/safety.py,sha256=mkFDO_BeEgiKjQd9o2I4UxB6XI7a9U-oOFjZ8LGRUC4,1238
 sunholo/invoke/__init__.py,sha256=VOpsfhNf98az3at7bMaY1Fiw4UIZAS6zMmSqWd6_ksI,141
 sunholo/invoke/async_class.py,sha256=uvCP8ekUCfTRWk7xwofTzRqFykMAwrRZ4Ce_YUR6PVs,2820
@@ -94,7 +94,7 @@ sunholo/invoke/direct_vac_func.py,sha256=fuTJlH5PsqWhN_yVMaWisHCTZU1JEUz8I8yVbWs
 sunholo/invoke/invoke_vac_utils.py,sha256=sJc1edHTHMzMGXjji1N67c3iUaP7BmAL5nj82Qof63M,2053
 sunholo/langfuse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sunholo/langfuse/callback.py,sha256=jl0SZsFS53uMW9DGeM9SOL_EsRZsba0wwFGLqKzu9_U,1684
-sunholo/langfuse/evals.py,sha256=y1T6qUA-k41BD2_5warjd1CQN4N107zsDizg9j5jCW4,3412
+sunholo/langfuse/evals.py,sha256=MNV3YaZ0b46WcuGZaurSVitWpFpbNaceiB47A2Zynjg,3789
 sunholo/langfuse/prompts.py,sha256=27BsVfihM6-h1jscbkGSO4HsATl-d4ZN6tcNCVztWoY,1300
 sunholo/llamaindex/__init__.py,sha256=DlY_cHWCsVEV1C5WBgDdHRgOMlJc8pDoCRukUJ8PT9w,88
 sunholo/llamaindex/get_files.py,sha256=6rhXCDqQ_lrIapISQ_OYQDjiSATXvS_9m3qq53-oIl0,781
@@ -144,9 +144,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=ZirFbS7EKxQaoRmOrb4BnG6jPJ83wt43N8M4zTVbutU,7717
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.95.4.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.95.4.dist-info/METADATA,sha256=0uNENTJHgisWBlLoAF-E2HpFxgxWiuAhxpJ9Tw9-LbE,7889
-sunholo-0.95.4.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-sunholo-0.95.4.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.95.4.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.95.4.dist-info/RECORD,,
+sunholo-0.96.0.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.96.0.dist-info/METADATA,sha256=vNdFJP76HWDA62pJ1med8qqeKi1Cp1IuzDPgjfBZ2-Q,7889
+sunholo-0.96.0.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+sunholo-0.96.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.96.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.96.0.dist-info/RECORD,,