llumo 0.2.22__py3-none-any.whl → 0.2.24__py3-none-any.whl

llumo/client.py

@@ -99,11 +99,23 @@ class LlumoClient:
             )
             self.email = data["data"]["data"].get("email", None)
 
-            self.definationMapping[evalName] = data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, None)
-
+            self.definationMapping[evalName] = data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, "")
+            self.categories = data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, "").get("categories", {})
+            self.evaluationStrictness=data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, "").get("evaluationStrictness", {})
+            self.grammarCheckOutput=data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, "").get("grammarCheckOutput", {})
+            self.insightsLength=data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, "").get("insightsLength", {})
+            self.insightsLevel=data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, "").get("insightsLevel", {})
+            self.executionDependency=data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, "").get("executionDependency", {})
+            self.sampleData=data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, "").get("sampleData", {})
+            self.numJudges=data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, "").get("numJudges", {})
+            self.penaltyBonusInstructions=data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, "").get("penaltyBonusInstructions", [])
+            self.probableEdgeCases= data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, "").get("probableEdgeCases", [])
+            self.fieldMapping= data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, "").get("fieldMapping", [])
+           
+            
         except Exception as e:
             # print(f"Error extracting data from response: {str(e)}")
-            raise LlumoAIError.UnexpectedError(detail=str(e))
+            raise LlumoAIError.UnexpectedError(detail=evalName)
 
     def postBatch(self, batch, workspaceID):
         payload = {
@@ -679,8 +691,9 @@ class LlumoClient:
 
             # Validate API and dependencies
             self.validateApiKey(evalName=evalName)
+            customAnalytics=getCustomAnalytics(self.workspaceID)
             metricDependencies = checkDependency(
-                evalName, list(dataframe.columns), tocheck=_tocheck
+                evalName, list(dataframe.columns), tocheck=_tocheck,customevals=customAnalytics
             )
             if not metricDependencies["status"]:
                 raise LlumoAIError.dependencyError(metricDependencies["message"])
@@ -691,6 +704,15 @@ class LlumoClient:
             evalType = "LLM"
             workspaceID = self.workspaceID
             email = self.email
+            categories=self.categories
+            evaluationStrictness=self.evaluationStrictness
+            grammarCheckOutput=self.grammarCheckOutput
+            insightLength=self.insightsLength
+            numJudges=self.numJudges
+            penaltyBonusInstructions=self.penaltyBonusInstructions
+            probableEdgeCases=self.probableEdgeCases
+            fieldMapping=self.fieldMapping
+            
 
             userHits = checkUserHits(
                 self.workspaceID,
@@ -720,6 +742,7 @@ class LlumoClient:
 
                 inputDict = {key: row[key] for key in keys if key in row}
                 output = row.get(outputColName, "")
+                intermediateSteps = row.get("intermediateSteps", "")
 
                 
                 rowID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
@@ -731,30 +754,39 @@ class LlumoClient:
                 templateData = {
                     "processID": getProcessID(),
                     "socketID": socketID,
+                    "rowID": rowID,
+                    "columnID": columnID,
+                    "processType": "EVAL",
+                    "evalType": evalType,
+                    "workspaceID": workspaceID,
+                    "email": email,
+                    "playgroundID": activePlayground,
                     "source": "SDK",
                     "processData": {
+                        "analyticsName": evalName,
+                        "definition": evalDefinition,
                         "executionDependency": {
-                            "query": "",
-                            "context": "",
-                            "output": output,
-                            "tools": tools,
-                            "groundTruth": groundTruth,
-                            "messageHistory": messageHistory,
+                            "Query": "",
+                            "Context": "",
+                            "Output": output,
+                            "Tools": tools,
+                            "GroundTruth": groundTruth,
+                            "MessageHistory": messageHistory,
+                            "IntermediateSteps": intermediateSteps,
                         },
-                        "definition": evalDefinition,
+                        "categories":categories,
+                        "evaluationStrictness": evaluationStrictness,
+                        "grammarCheckOutput": grammarCheckOutput,
+                        "insightLength": insightLength,
+                        "numJudges": numJudges,
+                        "penaltyBonusInstructions": penaltyBonusInstructions,
+                        "probableEdgeCases": probableEdgeCases,
                         "model": model,
                         "provider": provider,
-                        "analytics": evalName,
                     },
-                    "workspaceID": workspaceID,
                     "type": "EVAL",
-                    "evalType": evalType,
                     "kpi": evalName,
-                    "columnID": columnID,
-                    "rowID": rowID,
-                    "playgroundID": activePlayground,
-                    "processType": "EVAL",
-                    "email": email,
+                    "fieldMappig":fieldMapping,
                 }
 
                 query = ""
@@ -826,7 +858,6 @@ class LlumoClient:
         # print("All received keys:", received_rowIDs)
         # print("Missing keys:", len(missingRowIDs))
         missingRowIDs=list(missingRowIDs)
-        
         if len(missingRowIDs) > 0:
             dataFromDb=fetchData(workspaceID,activePlayground,missingRowIDs)
             rawResults.extend(dataFromDb)
@@ -866,11 +897,18 @@ class LlumoClient:
                 promptText=prompt_template,
                 definationMapping=self.definationMapping,
                 outputColName=outputColName,
-                activePlayground= activePlayground
+                activePlayground= activePlayground,
+                customAnalytics=customAnalytics
             ):
                 print(
                     "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://app.llumo.ai/evallm to see the results."
                 )
+                if getDataFrame:
+                    return LlumoDataFrameResults(dataframe,evals=self.evals,evalData=self.evalData,definationMapping=self.definationMapping)
+                else:
+                    data=dataframe.to_dict(orient="records")
+                    return LlumoDictResults(data,evals=self.evals,evalData=self.evalData,definationMapping=self.definationMapping)
+
         else:
             if getDataFrame:
                 return LlumoDataFrameResults(dataframe,evals=self.evals,evalData=self.evalData,definationMapping=self.definationMapping)
@@ -935,6 +973,7 @@ class LlumoClient:
             ):
                 print(
                     "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://app.llumo.ai/evallm to see the results.")
+                
         else:
             if getDataFrame == True and toEvaluate == True:
                 return LlumoDataFrameResults(working_df, evals=self.evals, evalData=self.evalData,
@@ -1231,6 +1270,20 @@ class LlumoClient:
             ):
                 print(
                     "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://app.llumo.ai/evallm to see the results.")
+                if getDataFrame == True and toEvaluate == True:
+                    return LlumoDataFrameResults(working_df, evals=self.evals, evalData=self.evalData,
+                                             definationMapping=self.definationMapping)
+
+                elif getDataFrame == False and toEvaluate == True:
+                    data = working_df.to_dict(orient="records")
+                    return LlumoDictResults(data, evals=self.evals, evalData=self.evalData,
+                                            definationMapping=self.definationMapping)
+
+                elif getDataFrame== True and toEvaluate == False:
+                    return working_df
+
+                elif getDataFrame == False and toEvaluate == False :
+                    return working_df.to_dict(orient = "records")
         else:
             if getDataFrame == True and toEvaluate == True:
                 return LlumoDataFrameResults(working_df, evals=self.evals, evalData=self.evalData,
@@ -1468,6 +1521,12 @@ class LlumoClient:
             ):
                 print(
                     "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://app.llumo.ai/evallm to see the results.")
+                if getDataFrame:
+                    return working_df
+
+                else:
+                    data = working_df.to_dict(orient="records")
+                    return data
         else:
             if getDataFrame:
                 return working_df

llumo/exceptions.py

@@ -22,8 +22,8 @@ class LlumoAIError(Exception):
         return LlumoAIError("The API response is not in valid JSON format")
 
     @staticmethod
-    def UnexpectedError(detail="An UnexpectedError error occurred"):
-        return LlumoAIError(f"UnexpectedError error: {detail}")
+    def UnexpectedError(detail="Metric"):
+        return LlumoAIError(f"Can you please check if {detail} is written correctly. If you want to run {detail} please create a custom eval with same name of app.llumo.ai/evallm ")
 
     @staticmethod
     def EvalError(detail="Some error occured while processing"):

llumo/helpingFuntions.py

@@ -26,6 +26,7 @@ uploadRowList = (
 )
 createInsightUrl="https://app.llumo.ai/api/external/generate-insight-from-eval-for-sdk"
 
+getCustomAnalyticsUrl="https://app.llumo.ai/api/workspace/get-all-analytics"
 
 def getProcessID():
     return f"{int(time.time() * 1000)}{uuid.uuid4()}"
@@ -217,7 +218,7 @@ def deleteColumnListInPlayground(workspaceID: str, playgroundID: str):
         return None
 
 def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColName=None,
-                 outputColName= "output",dataStreamName=None,definationMapping=None,evalOutputMap = None):
+                 outputColName= "output",dataStreamName=None,definationMapping=None,evalOutputMap = None,customAnalytics=[]):
     if len(dataframe) > 100:
         dataframe = dataframe.head(100)
         print("⚠️ Dataframe truncated to 100 rows for upload.")
@@ -227,10 +228,13 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
         "playgroundID": playgroundID,
         "columnListToUpload": [],
     }
-    allEvals = ['Response Completeness', 'Response Bias', 'Response Harmfulness', 'Input Toxicity', 'Input Harmfulness', 'Context Utilization', 'Relevance Retention', 'Semantic Cohesion', 'Final Task Alignment', 'Tool Reliability', 'Response Correctness', 'Response Toxicity', 'Input Bias', 'Input Relevancy', 'Redundancy Reduction', 'Response Sentiment', 'Tool Selection Accuracy', 'Stepwise Progression', 'Hallucination', 'Faithfulness', 'Answer Relevancy', 'Context Precision', 'Answer Similarity', 'Harmfulness', 'Maliciousness', 'Coherence', 'Answer Correctness', 'Context Recall', 'Context Entity Recall', 'Conciseness', 'customEvalColumn', 'Groundedness', 'Memory Utilization', 'Input Relevancy (Multi-turn)']
-
-    evalDependencies =  checkDependency(_returnDepMapping=True)
-
+    allEvals = ['Response Completeness', 'Response Bias', 'Response Harmfulness', 'Input Toxicity', 'Input Harmfulness', 'Context Utilization', 'Relevance Retention', 'Semantic Cohesion', 'Final Task Alignment', 'Tool Reliability', 'Response Correctness', 'Response Toxicity', 'Input Bias', 'Input Relevancy', 'Redundancy Reduction', 'Response Sentiment', 'Tool Selection Accuracy', 'Stepwise Progression', 'Hallucination', 'Faithfulness', 'Answer Relevancy', 'Context Precision', 'Answer Similarity', 'Harmfulness', 'Maliciousness', 'Coherence', 'Answer Correctness', 'Context Recall', 'Context Entity Recall', 'Conciseness', 'customEvalColumn', 'Groundedness', 'Memory Utilization', 'Input Relevancy (Multi-turn)','PII Check','Prompt Injection']
+    try:
+        allEvals.extend(list(customAnalytics.keys()))
+    except Exception as e:
+        pass
+    evalDependencies =  checkDependency(_returnDepMapping=True,customevals=customAnalytics)
+    print(allEvals)
     # Create a mapping of column names to unique column IDs
     columnIDMapping = {}
 
@@ -296,7 +300,7 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
                     "order": indx}
 
 
-        elif any(col.startswith(eval + "_") or col == eval for eval in allEvals) and not " Reason" in col and promptText is not None:
+        elif any(col.startswith(eval + "_") or col == eval for eval in allEvals) and not " Reason" in col and promptText is not None :
             if evalOutputMap != None:
                 outputColName = evalOutputMap[col]
             else:
@@ -455,14 +459,14 @@ def uploadRowsInDBPlayground(payload):
         return None
 
 
-def createPlayground(email, workspaceID, df, promptText=None,queryColName=None,dataStreamName=None,definationMapping=None,outputColName="output",evalOutputMap = None,activePlayground=None):
+def createPlayground(email, workspaceID, df, promptText=None,queryColName=None,dataStreamName=None,definationMapping=None,outputColName="output",evalOutputMap = None,activePlayground=None,customAnalytics=[]):
     
     if activePlayground != None:
         playgroundId=activePlayground
     else:
         playgroundId = str(createEvalPlayground(email=email, workspaceID=workspaceID))
     payload1, payload2 = createColumn(
-        workspaceID=workspaceID, dataframe=df, playgroundID=playgroundId, promptText=promptText,queryColName=queryColName,dataStreamName=dataStreamName,definationMapping=definationMapping,outputColName=outputColName,evalOutputMap=evalOutputMap
+        workspaceID=workspaceID, dataframe=df, playgroundID=playgroundId, promptText=promptText,queryColName=queryColName,dataStreamName=dataStreamName,definationMapping=definationMapping,outputColName=outputColName,evalOutputMap=evalOutputMap,customAnalytics=customAnalytics
     )
     
  # Debugging line to check the payload2 structure
@@ -515,7 +519,7 @@ def getPlaygroundInsights(defination:str,uniqueClassesString: str, reasonList: l
     else:
         print(f"Error generating insight: {responseGenerate.status_code} - {responseGenerate.text}")
         return None
-def checkDependency(selectedEval:list = [], columns:list = [],tocheck=True,_returnDepMapping = False):
+def checkDependency(selectedEval:list = [], columns:list = [],tocheck=True,_returnDepMapping = False,customevals={}):
     """
     Checks if all the required input columns for the selected evaluation metric are present.
 
@@ -527,6 +531,7 @@ def checkDependency(selectedEval:list = [], columns:list = [],tocheck=True,_retu
     - LlumoAIError.dependencyError: If any required column is missing.
     """
     # Define required dependencies for each evaluation metric
+
     metricDependencies = {
         'Response Completeness': ['context', 'query', 'output'],
         'Response Bias': ['output'],
@@ -537,20 +542,24 @@ def checkDependency(selectedEval:list = [], columns:list = [],tocheck=True,_retu
         'Relevance Retention': ['context', 'query'],
         'Semantic Cohesion': ['context'],
         'Final Task Alignment': ['query','output'],
-        'Tool Reliability': ['messageHistory'],
+        'Tool Reliability': ['intermediateSteps'],
         'Response Correctness': ['output', 'query', 'context'],
         'Response Toxicity': ['output'],
         'Input Bias': ['query'],
         'Input Relevancy': ['context', 'query'],
         'Redundancy Reduction': ['context'],
         'Response Sentiment': ['output'],
-        'Tool Selection Accuracy': ['tools', 'messageHistory'],
-        'Stepwise Progression': ['tools', 'messageHistory'],
+        'Tool Selection Accuracy': ['tools', 'intermediateSteps'],
+        'Stepwise Progression': ['tools', 'intermediateSteps'],
         'Hallucination': ['query', 'context', 'output'],
         'Groundedness': ['groundTruth', 'output'],
         'Memory Utilization': ['context', 'messageHistory'],
-        'Input Relevancy (Multi-turn)': ['context', 'query']
+        'Input Relevancy (Multi-turn)': ['context', 'query'],
+        'PII Check':["query","output"],
+        'Prompt Injection':["query"]
     }
+    
+    metricDependencies.update(customevals)
     if _returnDepMapping == True:
         return metricDependencies
 
@@ -650,6 +659,7 @@ def validateGoogleKey(api_key):
 
 def groupLogsByClass(logs, max_logs=2):
     # Initialize the final result structures (no defaultdict)
+    
     groupedLogs = {}
     uniqueEdgeCases = {}  # This will store unique edge cases for each eval_name
 
@@ -683,3 +693,46 @@ def groupLogsByClass(logs, max_logs=2):
         uniqueEdgeCases[eval_name] = list(uniqueEdgeCases[eval_name])
 
     return groupedLogs, uniqueEdgeCases
+
+
+def getCustomAnalytics(workspaceID):
+    try:
+        url = getCustomAnalyticsUrl
+        payload = {
+            "workspaceID": workspaceID
+        }
+
+        headers = {
+            "Content-Type": "application/json"
+        }
+
+        response = requests.post(url, json=payload, headers=headers)
+        data=response.json()
+        customData=data.get("data","").get("analyticsCustom","")
+        customMapping = {
+            "QUERY": "query",
+            "CONTEXT": "context",
+            "OUTPUT": "output",
+            "MESSAGEHISTORY": "messageHistory",
+            "TOOLS": "tools",
+            "INTERMEDIATESTEPS": "intermediateSteps",
+            "GROUNDTRUTH": "groundTruth",
+        }
+
+        metricDependencies = {}
+        
+        
+
+        for eval in customData:
+            evalName = eval.get("analyticsName")
+            evalDependencyRaw = list(eval.get("variableMappings").values())
+            
+            # Replace each value using the custom mapping
+            evalDependency = [customMapping.get(val.upper(), val.lower()) for val in evalDependencyRaw]
+            
+            # Build the dict
+            metricDependencies[evalName] = evalDependency
+        return metricDependencies
+    
+    except Exception as e:
+        return {}

{llumo-0.2.22.dist-info → llumo-0.2.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.22
+Version: 0.2.24
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

{llumo-0.2.22.dist-info → llumo-0.2.24.dist-info}/RECORD

@@ -1,16 +1,16 @@
 llumo/__init__.py,sha256=YVBkF1fiXFBd_zzySi9BDWgX8MJuLBJ-oF8538MrnDU,256
 llumo/chains.py,sha256=6lCgLseh04RUgc6SahhmvQj82quay2Mi1j8gPUlx8Es,2923
-llumo/client.py,sha256=HuqYBLvqWfJA5HRkMqxpiEsTiLbVCv6mL4t6guRROWU,63329
-llumo/exceptions.py,sha256=Vp_MnanHbnd1Yjuoi6WLrKiwwZbJL3znCox2URMmGU4,2032
+llumo/client.py,sha256=pCocD7v5dDuHKlWE6OqZrrEk4e_LWYuI_LgWaGT3E-g,67498
+llumo/exceptions.py,sha256=1OyhN9YL9LcyUPUsqYHq6Rret0udATZAwMVJaio2_Ec,2123
 llumo/execution.py,sha256=nWbJ7AvWuUPcOb6i-JzKRna_PvF-ewZTiK8skS-5n3w,1380
 llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
 llumo/google.py,sha256=3S_aRtbtlctCXPGR0u4baLlkyFrsjd02vlUCkoRPA5U,2147
-llumo/helpingFuntions.py,sha256=DDvvr1LO57uMZQekncYrv8IpE4QDG2qf9pBP6DIvrLU,25523
+llumo/helpingFuntions.py,sha256=B6FwUQ5f1v4FKrWCbYoGWMFdscOV_liuuhTgNQ3cdrk,27275
 llumo/models.py,sha256=aVEZsOOoQx5LeNtwSyBxqvrINq0izH3QWu_YjsMPE6o,2910
 llumo/openai.py,sha256=DGhEwQIJIIycGpw3hYQnyxdj6RFVpZ-gay-fZGqtkhU,3013
 llumo/sockets.py,sha256=pBDo-U65hMIMwKMwZQl3iBkEjISEt-9BkXxZTWfSHF4,6116
-llumo-0.2.22.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
-llumo-0.2.22.dist-info/METADATA,sha256=Xodp45eD66Ym9HGYaGahLGpaCQL1A-eDPZY0z7tXx38,1558
-llumo-0.2.22.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-llumo-0.2.22.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
-llumo-0.2.22.dist-info/RECORD,,
+llumo-0.2.24.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
+llumo-0.2.24.dist-info/METADATA,sha256=IfYBa5UYiXuF595u13Qyi5MoQorHzxpfpUSPmciS7rM,1558
+llumo-0.2.24.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+llumo-0.2.24.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
+llumo-0.2.24.dist-info/RECORD,,