llumo 0.2.13__py3-none-any.whl → 0.2.14b1__py3-none-any.whl

llumo/client.py

@@ -80,17 +80,16 @@ class LlumoClient:
             raise LlumoAIError.InvalidApiResponse()
 
         try:
-            self.hitsAvailable = data["data"]["data"].get("remainingHits", 0)
+            self.hitsAvailable = data['data']["data"].get("remainingHits", 0)
             self.workspaceID = data["data"]["data"].get("workspaceID")
-            self.evalDefinition = data["data"]["data"].get("analyticsMapping")
+            self.evalDefinition = data["data"]["data"]["analyticsMapping"]
             self.socketToken = data["data"]["data"].get("token")
             self.hasSubscribed = data["data"]["data"].get("hasSubscribed", False)
             self.trialEndDate = data["data"]["data"].get("trialEndDate", None)
             self.subscriptionEndDate = data["data"]["data"].get("subscriptionEndDate", None)
             self.email = data["data"]["data"].get("email", None)
             
-            self.definationMapping[evalName] = self.evalDefinition
-
+            self.definationMapping[evalName] = data["data"]["data"]["analyticsMapping"][evalName]
         except Exception as e:
             # print(f"Error extracting data from response: {str(e)}")
             raise LlumoAIError.UnexpectedError(detail=str(e))
@@ -159,12 +158,21 @@ class LlumoClient:
     # this function allows the users to run exactl one eval at a time
     def evaluate(
         self,
-        dataframe,
+        data,
         eval="Response Completeness",
         prompt_template="",
         outputColName="output",
         createExperiment: bool = False,
+        _tocheck = True,
     ):
+        
+        # converting it into a pandas dataframe object
+        dataframe = pd.DataFrame(data)
+
+        # check for dependencies for the selected eval metric
+        metricDependencies = checkDependency(eval,columns=list(dataframe.columns),tocheck=_tocheck)
+        if metricDependencies["status"] == False:
+            raise LlumoAIError.dependencyError(metricDependencies["message"])
 
         results = {}
         try:
@@ -206,7 +214,7 @@ class LlumoClient:
             # if self.hitsAvailable == 0 or len(dataframe) > self.hitsAvailable:
             #     raise LlumoAIError.InsufficientCredits()
 
-            evalDefinition = self.evalDefinition[eval]
+            evalDefinition = self.evalDefinition[eval].get("definition")
             model = "GPT_4"
             provider = "OPENAI"
             evalType = "LLM"
@@ -324,7 +332,7 @@ class LlumoClient:
                 except Exception as e:
                     continue
 
-                time.sleep(1)
+                time.sleep(3)
 
             timeout = max(50, min(600, totalItems * 10))
 
@@ -377,31 +385,71 @@ class LlumoClient:
     # this function allows the users to run multiple evals at once
     def evaluateMultiple(
             self,
-            dataframe,
+            data,
             eval=["Response Completeness"],
-            prompt_template="",
+            prompt_template="Give answer to the given query:{{query}} , using the given context: {{context}}",
             outputColName="output",
             createExperiment: bool = False,
+            _tocheck = True,
     ):
-        resultdf  = dataframe.copy()
+        """
+        Runs multiple evaluation metrics on the same input dataset.
+
+        Parameters:
+            data (list of dict): Input data, where each dict represents a row.
+            eval (list of str): List of evaluation metric names to run.
+            prompt_template (str): Optional prompt template used in evaluation.
+            outputColName (str): Column name in data that holds the model output.
+            createExperiment (bool): Whether to log the results to Llumo playground.
+
+        Returns:
+            pandas.DataFrame: Final dataframe with all evaluation results.
+        """
+        
+        # Convert input dict list into a DataFrame
+        dataframe = pd.DataFrame(data)
+
+        # Copy to hold final results
+        resultdf = dataframe.copy()
+
+        # Run each evaluation metric one by one
         for evalName in eval:
-            resultdf = self.evaluate(dataframe = resultdf,eval=evalName,prompt_template=prompt_template,outputColName=outputColName,createExperiment = False)
+            time.sleep(2)  # small delay to avoid overload or rate limits
+
+            # Call evaluate (assumes evaluate takes dict, not dataframe)
+            resultdf = self.evaluate(
+                data=resultdf.to_dict(orient="records"),  # convert df back to dict list
+                eval=evalName,
+                prompt_template=prompt_template,
+                outputColName=outputColName,
+                createExperiment=False,
+                _tocheck=_tocheck,
+            )
 
+        # Save to playground if requested
         if createExperiment:
             pd.set_option("future.no_silent_downcasting", True)
             df = resultdf.fillna("Some error occured").astype(object)
 
-            if createPlayground(self.email, self.workspaceID, df,definationMapping=self.definationMapping,outputColName=outputColName,promptText=prompt_template):
+            if createPlayground(
+                    self.email,
+                    self.workspaceID,
+                    df,
+                    definationMapping=self.definationMapping,
+                    outputColName=outputColName,
+                    promptText=prompt_template
+            ):
                 print(
-                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results.Please rerun the experiment to see the results on playground."
+                    "Your data has been saved in the Llumo Experiment. "
+                    "Visit https://app.llumo.ai/evallm to see the results. "
+                    "Please rerun the experiment to see the results on playground."
                 )
         else:
             return resultdf
 
-
-    def evaluateCompressor(self, dataframe, prompt_template):
+    def evaluateCompressor(self, data, prompt_template):
         results = []
-
+        dataframe = pd.DataFrame(data)
         try:
             socketID = self.socket.connect(timeout=150)
             max_wait_secs = 20
@@ -608,10 +656,11 @@ class LlumoClient:
                 outputColName = f"output_{i}"
                 try:
                     res = self.evaluateMultiple(
-                        df,
+                        df.to_dict("records"),
                         eval=eval,
                         prompt_template=str(templates[0]),
                         outputColName=outputColName,
+                        _tocheck=False,
                     )
 
                     # Rename all new columns with _i+1 (e.g., _1, _2)
@@ -633,7 +682,7 @@ class LlumoClient:
             if createExperiment:
                 pd.set_option("future.no_silent_downcasting", True)
                 dfWithEvals = dfWithEvals.fillna("Some error occurred")
-                if createPlayground(email, workspaceID, dfWithEvals, promptText=templates[0]):
+                if createPlayground(email, workspaceID, dfWithEvals, promptText=templates[0],definationMapping=self.definationMapping):
                     
                     print("Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results.")
             else:
@@ -652,33 +701,37 @@ class LlumoClient:
     # this function generates an output using llm and tools and evaluate that output
     def evaluateAgents(
         self,
-        dataframe,
+        data,
         model,
         agents,
         model_api_key=None,
+        evals=["Final Task Alignment"],
         prompt_template="Give answer for the given query: {{query}}",
         createExperiment: bool = False,
     ):
         if model.lower() not in ["openai", "google"]:
             raise ValueError("Model must be 'openai' or 'google'")
 
+        # converting into pandas dataframe object
+        dataframe = pd.DataFrame(data)
+
         # Run unified agent execution
         toolResponseDf = LlumoAgentExecutor.run(
             dataframe, agents, model=model, model_api_key=model_api_key
         )
         
         
-        evals = [
-            "Tool Reliability",
-            "Stepwise Progression",
-            "Tool Selection Accuracy",
-            "Final Task Alignment",
-        ]
+        # evals = [
+        #     "Tool Reliability",
+        #     "Stepwise Progression",
+        #     "Tool Selection Accuracy",
+        #     "Final Task Alignment",
+        # ]
 
         for eval in evals:
             # Perform evaluation
             toolResponseDf = self.evaluate(
-                toolResponseDf,
+                toolResponseDf.to_dict(orient = "records"),
                 eval=eval,
                 prompt_template=prompt_template,
                 createExperiment=False,
@@ -696,32 +749,34 @@ class LlumoClient:
     # this function evaluate that tools output given by the user
     def evaluateAgentResponses(
         self,
-        dataframe,
+        data,
         evals=["Final Task Alignment"],
         outputColName="output",
         createExperiment: bool = False,
     ):
+        dataframe = pd.DataFrame(data)
+
         try:
             if "query" and "messageHistory" and "tools" not in dataframe.columns:
                 raise ValueError(
                     "DataFrame must contain 'query', 'messageHistory','output' ,and 'tools' columns. Make sure the columns names are same as mentioned here."
                 )
-            prompt_template="Give answer for the given query: {{query}}"
-            
+
+
             # evals = [
             #     "Tool Reliability",
             #     "Stepwise Progression",
             #     "Tool Selection Accuracy",
             #     "Final Task Alignment",
             # ]
+
             toolResponseDf = dataframe.copy()
             for eval in evals:
                 # Perform evaluation
                 toolResponseDf = self.evaluate(
-                    toolResponseDf, eval=eval, prompt_template=prompt_template,outputColName=outputColName
+                    toolResponseDf.to_dict(orient = "records"), eval=eval, prompt_template="Give answer for the given query: {{query}}",outputColName=outputColName
                 )
-                
-           
+
             return toolResponseDf
 
         except Exception as e:

llumo/exceptions.py

@@ -46,6 +46,10 @@ class LlumoAIError(Exception):
     def modelHitsExhausted(details = "Your credits for the selected model exhausted."):
         return LlumoAIError(details)
 
+    @staticmethod
+    def dependencyError(details):
+        return LlumoAIError(details)
+
     # @staticmethod
     # def dateNotFound():
     #     return LlumoAIError("Trial end date or subscription end date not found for the given user.")

llumo/helpingFuntions.py

@@ -303,8 +303,8 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
                     dependencies.append(columnIDMapping[varName])
                     
             dependencies.append(columnIDMapping[outputColName])  # Add the output column ID
-            longDef = definationMapping.get(col, {}).get(col, "")
-            shortDef ="You have run this from SDK"
+            longDef = definationMapping.get(col, {}).get('definition', "")
+            shortDef =definationMapping.get(col, {}).get('briefDefinition', "")
             enum =  col.upper().replace(" ","_")
 
             template = {
@@ -312,7 +312,7 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
         col.lower().replace(" ","_")
       ],
       "evaluationMetric": "ALL",
-      "evaluationModel": "GEMINI_PRO",
+      "evaluationModel": "LLUMO_EVALLM",
       "selectPrompt": columnIDMapping[outputColName],
       "scoreCondition": "GREATER_THAN",
       "scoreValue": "50",
@@ -336,7 +336,7 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
       "analyticsENUM": enum,
       "prompt": shortDef,
       "analyticsName": col,
-      "columnID": str(uuid.uuid4().hex[:8]),
+      "columnID": columnID,
       "label": col,
       "order": indx
     }
@@ -371,9 +371,24 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
 
         # For each column, we need to map the column ID to the corresponding value in the row
         for col in dataframe.columns:
-            columnID = columnIDMapping[col]  # Get the columnID from the mapping
-            row_dict[columnID] = row[col]  # Map the columnID to the value in the row
-
+            columnID = columnIDMapping[col]
+            
+            if col in allEvals:
+                row_dict[columnID] = {
+                    
+                    "value": row[col],
+                    "type": "EVAL",
+                    "isValid": True,
+                    "reasoning": row[col+" Reason"],
+                    "edgeCase": "minorHallucinationDetailNotInContext",
+                    "kpi": col
+                
+                    }
+            else:# Get the columnID from the mapping
+                row_dict[columnID] = {
+                    "value": row[col],
+                    "type":"VARIABLE"# Map the columnID to the value in the row
+                }
         # Add the row index (if necessary)
         row_dict["pIndex"] = indx
         rowTemplate["dataToUploadList"].append(row_dict)
@@ -430,6 +445,9 @@ def createPlayground(email, workspaceID, df, promptText=None,queryColName=None,d
     payload1, payload2 = createColumn(
         workspaceID=workspaceID, dataframe=df, playgroundID=playgroundId, promptText=promptText,queryColName=queryColName,dataStreamName=dataStreamName,definationMapping=definationMapping,outputColName=outputColName
     )
+    
+    
+    
     deleteExistingRows = deleteColumnListInPlayground(
         workspaceID=workspaceID, playgroundID=playgroundId
     )
@@ -490,4 +508,58 @@ def getPlaygroundInsights(workspaceID: str, activePlayground: str):
     else:
         print(f"Error generating insight: {responseGenerate.status_code} - {responseGenerate.text}")
         return None
-
+def checkDependency(selectedEval, columns,tocheck=True):
+    """
+    Checks if all the required input columns for the selected evaluation metric are present.
+
+    Parameters:
+    - selectedEval (str): The name of the selected evaluation metric.
+    - columns (list): List of column names present in the dataset.
+
+    Raises:
+    - LlumoAIError.dependencyError: If any required column is missing.
+    """
+    if tocheck:
+    # Define required dependencies for each evaluation metric
+        metricDependencies = {
+            'Response Completeness': ['context', 'query', 'output'],
+            'Response Bias': ['output'],
+            'Response Harmfulness': ['output'],
+            'Input Toxicity': ['query'],
+            'Input Harmfulness': ['query'],
+            'Context Utilization': ['output', 'context'],
+            'Relevance Retention': ['context', 'query'],
+            'Semantic Cohesion': ['context'],
+            'Final Task Alignment': ['messageHistory'],
+            'Tool Reliability': ['messageHistory'],
+            'Response Correctness': ['output', 'query', 'context'],
+            'Response Toxicity': ['output'],
+            'Input Bias': ['query'],
+            'Input Relevancy': ['context', 'query'],
+            'Redundancy Reduction': ['context'],
+            'Response Sentiment': ['output'],
+            'Tool Selection Accuracy': ['tools', 'messageHistory'],
+            'Stepwise Progression': ['tools', 'messageHistory'],
+            'Hallucination': ['query', 'context', 'output'],
+            'Groundedness': ['groundTruth', 'output'],
+            'Memory Utilization': ['context', 'messageHistory'],
+            'Input Relevancy (Multi-turn)': ['context', 'query']
+        }
+
+        # Check if the selected evaluation metric is known
+        if selectedEval not in metricDependencies:
+            return {"status": False,"message":f"Unknown evaluation metric: {selectedEval}"}
+
+        # Get the required columns for the selected evaluation
+        columnsRequired = metricDependencies[selectedEval]
+
+        # Check if each required column is present in the provided columns
+        for requirement in columnsRequired:
+            if requirement not in columns:
+                return {"status":False,
+                    "message":f"'{selectedEval}' requires columns: {columnsRequired}. "
+                    f"Missing: '{requirement}'. Please ensure your data includes all required columns."
+                    }
+        return {"status":True,"message":"success"}
+    else:
+        return {"status":True,"message":"success"}

llumo/sockets.py

@@ -101,8 +101,8 @@ class LlumoSocketClient:
             raise RuntimeError(f"WebSocket connection failed: {e}")
 
     def listenForResults(self, min_wait=30, max_wait=300, inactivity_timeout=50, expected_results=None):
-        if not self._connected:
-            raise RuntimeError("WebSocket is not connected. Call connect() first.")
+        # if not self._connected:
+        #     raise RuntimeError("WebSocket is not connected. Call connect() first.")
 
         self._expected_results = expected_results  # NEW
         start_time = time.time()

{llumo-0.2.13.dist-info → llumo-0.2.14b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.13
+Version: 0.2.14b1
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

llumo-0.2.14b1.dist-info/RECORD

@@ -0,0 +1,13 @@
+llumo/__init__.py,sha256=O04b4yW1BnOvcHzxWFddAKhtdBEhBNhLdb6xgnpHH_Q,205
+llumo/client.py,sha256=KbvBokbeINsEv2bJ9S3Zys6GYmpA7BIsjchmYZJx5C0,38133
+llumo/exceptions.py,sha256=i3Qv4_g7XjRuho7-b7ybjw2bwSh_NhvICR6ZAgiLQX8,1944
+llumo/execution.py,sha256=x88wQV8eL99wNN5YtjFaAMCIfN1PdfQVlAZQb4vzgQ0,1413
+llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
+llumo/helpingFuntions.py,sha256=FOHDxnofS316u6fnuPDVzM-fIrTzAGkMTAjoiadlKVw,20230
+llumo/models.py,sha256=YH-qAMnShmUpmKE2LQAzQdpRsaXkFSlOqMxHwU4zBUI,1560
+llumo/sockets.py,sha256=0BCcdCaiXDR7LO_9NIYA6urtpgdmyWW2M1US67G9Eus,5583
+llumo-0.2.14b1.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
+llumo-0.2.14b1.dist-info/METADATA,sha256=TxKTceUWuxS1UkZT1vd_3yVbR98_ntHSEU1yq2HW_bg,1493
+llumo-0.2.14b1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+llumo-0.2.14b1.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
+llumo-0.2.14b1.dist-info/RECORD,,

llumo-0.2.13.dist-info/RECORD

@@ -1,13 +0,0 @@
-llumo/__init__.py,sha256=O04b4yW1BnOvcHzxWFddAKhtdBEhBNhLdb6xgnpHH_Q,205
-llumo/client.py,sha256=mVqxNyYMbHlL9lnkpw_H6_n98WYO38ZcLZNnQX1TjUE,35948
-llumo/exceptions.py,sha256=iCj7HhtO_ckC2EaVBdXbAudNpuMDsYmmMEV5lwynZ-E,1854
-llumo/execution.py,sha256=x88wQV8eL99wNN5YtjFaAMCIfN1PdfQVlAZQb4vzgQ0,1413
-llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
-llumo/helpingFuntions.py,sha256=ah0FUQcRV3gfguvjQQ_aZzq59hpJttqAPJdjJVNYdFc,17110
-llumo/models.py,sha256=YH-qAMnShmUpmKE2LQAzQdpRsaXkFSlOqMxHwU4zBUI,1560
-llumo/sockets.py,sha256=i90l2rr08paa-ifKy2E5YMIS8r3yRBmu2CUOjhFKork,5579
-llumo-0.2.13.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
-llumo-0.2.13.dist-info/METADATA,sha256=W-ZVsTKhiPw6strlNQEqyufgOLfV9ZCjCpY6Dj3Qc94,1491
-llumo-0.2.13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-llumo-0.2.13.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
-llumo-0.2.13.dist-info/RECORD,,