llumo 0.2.12__py3-none-any.whl → 0.2.13__py3-none-any.whl

llumo/client.py

@@ -30,6 +30,7 @@ class LlumoClient:
         self.apiKey = api_key
         self.socket = LlumoSocketClient(socketUrl)
         self.processMapping = {}
+        self.definationMapping = {}
 
     def validateApiKey(self, evalName=" "):
         headers = {
@@ -87,6 +88,8 @@ class LlumoClient:
             self.trialEndDate = data["data"]["data"].get("trialEndDate", None)
             self.subscriptionEndDate = data["data"]["data"].get("subscriptionEndDate", None)
             self.email = data["data"]["data"].get("email", None)
+            
+            self.definationMapping[evalName] = self.evalDefinition
 
         except Exception as e:
             # print(f"Error extracting data from response: {str(e)}")
@@ -315,6 +318,7 @@ class LlumoClient:
 
             for cnt, batch in enumerate(self.allBatches):
                 try:
+                    
                     self.postBatch(batch=batch, workspaceID=workspaceID)
                     # print("Betch Posted with item len: ", len(batch))
                 except Exception as e:
@@ -363,9 +367,9 @@ class LlumoClient:
             pd.set_option("future.no_silent_downcasting", True)
             df = dataframe.fillna("Some error occured").astype(object)
 
-            if createPlayground(email, workspaceID, df):
+            if createPlayground(email, workspaceID, df,promptText=prompt_template,definationMapping=self.definationMapping,outputColName=outputColName):
                 print(
-                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
+                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results.Please rerun the experiment to see the results on playground."
                 )
         else:
             return dataframe
@@ -387,9 +391,9 @@ class LlumoClient:
             pd.set_option("future.no_silent_downcasting", True)
             df = resultdf.fillna("Some error occured").astype(object)
 
-            if createPlayground(self.email, self.workspaceID, df):
+            if createPlayground(self.email, self.workspaceID, df,definationMapping=self.definationMapping,outputColName=outputColName,promptText=prompt_template):
                 print(
-                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
+                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results.Please rerun the experiment to see the results on playground."
                 )
         else:
             return resultdf
@@ -546,21 +550,22 @@ class LlumoClient:
         # dataframe["cost_saving"] = cost_saving
 
         return dataframe
-
     def run_sweep(
-        self,
-        templates: List[str],
-        dataset: Dict[str, List[str]],
-        model_aliases: List[AVAILABLEMODELS],
-        apiKey: str,
-        eval=["Response Correctness"],
-        toEvaluate: bool = False,
-        createExperiment: bool = False,
-    ) -> pd.DataFrame:
+    self,
+    templates: List[str],
+    dataset: Dict[str, List[str]],
+    model_aliases: List[AVAILABLEMODELS],
+    apiKey: str,
+    eval=["Response Correctness"],
+    toEvaluate: bool = False,
+    createExperiment: bool = False,
+) -> pd.DataFrame:
+        
         try:
             self.validateApiKey()
         except Exception as e:
-            raise "Some error ocuured please check your API key"
+            raise Exception("Some error occurred, please check your API key")
+
         workspaceID = self.workspaceID
         email = self.email
         executor = ModelExecutor(apiKey)
@@ -570,57 +575,80 @@ class LlumoClient:
 
         results = []
 
-        # Iterate through combinations
         for combo in combinations:
             for template in templates:
                 prompt = template
                 for k, v in combo.items():
                     prompt = prompt.replace(f"{{{{{k}}}}}", v)
-                # Add a row for each model
-                for model in model_aliases:
-                    row = {
-                        "template": template,
-                        "prompt": prompt,
-                        **combo,
-                        "model": model.value,
-                    }
 
+                row = {
+                    "prompt": prompt,
+                    **combo,
+                }
+
+                for i, model in enumerate(model_aliases, 1):
                     try:
                         provider = getProviderFromModel(model)
-                        response = executor.execute(
-                            provider, model.value, prompt, apiKey
-                        )
-                        row["output"] = response
+                        response = executor.execute(provider, model.value, prompt, apiKey)
+                        outputKey = f"output_{i}"
+                        row[outputKey] = response
                     except Exception as e:
-                        row["output"] = f"Error: {str(e)}"
+                        row[f"output_{i}"] = str(e)
 
-                    results.append(row)
+                results.append(row)
+
+        
+        
         df = pd.DataFrame(results)
-        if toEvaluate:
 
-            res = self.evaluateMultiple(df, eval=eval, prompt_template=str(templates[0]))
+        
+        if toEvaluate==True:
+            dfWithEvals = df.copy()
+            for i, model in enumerate(model_aliases,1):
+                outputColName = f"output_{i}"
+                try:
+                    res = self.evaluateMultiple(
+                        df,
+                        eval=eval,
+                        prompt_template=str(templates[0]),
+                        outputColName=outputColName,
+                    )
+
+                    # Rename all new columns with _i+1 (e.g., _1, _2)
+                    for evalMetric in eval:
+                        scoreCol = f"{evalMetric}"
+                        reasonCol = f"{evalMetric} Reason"
+                        if scoreCol in res.columns:
+                            res = res.rename(columns={scoreCol: f"{scoreCol}_{i}"})
+                        if reasonCol in res.columns:
+                            res = res.rename(columns={reasonCol: f"{reasonCol}_{i}"})
+
+                    # Drop duplicated columns from df (like prompt, variables, etc.)
+                    newCols = [col for col in res.columns if col not in dfWithEvals.columns]
+                    dfWithEvals = pd.concat([dfWithEvals, res[newCols]], axis=1)
+
+                except Exception as e:
+                    print(f"Evaluation failed for model {model.value}: {str(e)}")
 
             if createExperiment:
                 pd.set_option("future.no_silent_downcasting", True)
-                res = res.fillna("Some error occured")
-                if createPlayground(email, workspaceID, res):
-                    print(
-                        "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
-                    )
+                dfWithEvals = dfWithEvals.fillna("Some error occurred")
+                if createPlayground(email, workspaceID, dfWithEvals, promptText=templates[0]):
+                    
+                    print("Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results.")
             else:
-                return res
-
+                return dfWithEvals
         else:
-            if createExperiment:
+            if createExperiment==True:
                 pd.set_option("future.no_silent_downcasting", True)
-                df = df.fillna("Some error occured")
-                if createPlayground(email, workspaceID, df):
-                    print(
-                        "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
-                    )
-            else:
+                df = df.fillna("Some error occurred")
+
+                if createPlayground(email, workspaceID, df, promptText=templates[0]):
+                    print("Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results.")
+            else :
                 return df
 
+
     # this function generates an output using llm and tools and evaluate that output
     def evaluateAgents(
         self,
@@ -638,6 +666,8 @@ class LlumoClient:
         toolResponseDf = LlumoAgentExecutor.run(
             dataframe, agents, model=model, model_api_key=model_api_key
         )
+        
+        
         evals = [
             "Tool Reliability",
             "Stepwise Progression",
@@ -667,31 +697,37 @@ class LlumoClient:
     def evaluateAgentResponses(
         self,
         dataframe,
-        prompt_template="Give answer for the given query: {{query}}",
+        evals=["Final Task Alignment"],
+        outputColName="output",
         createExperiment: bool = False,
     ):
         try:
             if "query" and "messageHistory" and "tools" not in dataframe.columns:
                 raise ValueError(
-                    "DataFrame must contain 'query', 'messageHistory', and 'tools' columns"
+                    "DataFrame must contain 'query', 'messageHistory','output' ,and 'tools' columns. Make sure the columns names are same as mentioned here."
                 )
-            evals = [
-                "Tool Reliability",
-                "Stepwise Progression",
-                "Tool Selection Accuracy",
-                "Final Task Alignment",
-            ]
+            prompt_template="Give answer for the given query: {{query}}"
+            
+            # evals = [
+            #     "Tool Reliability",
+            #     "Stepwise Progression",
+            #     "Tool Selection Accuracy",
+            #     "Final Task Alignment",
+            # ]
             toolResponseDf = dataframe.copy()
             for eval in evals:
                 # Perform evaluation
                 toolResponseDf = self.evaluate(
-                    toolResponseDf, eval=eval, prompt_template=prompt_template
+                    toolResponseDf, eval=eval, prompt_template=prompt_template,outputColName=outputColName
                 )
+                
+           
             return toolResponseDf
 
         except Exception as e:
             raise e
 
+        
     def runDataStream(
         self,
         dataframe,
@@ -852,7 +888,7 @@ class LlumoClient:
             pd.set_option("future.no_silent_downcasting", True)
             df = dataframe.fillna("Some error occured").astype(object)
 
-            if createPlayground(email, workspaceID, df):
+            if createPlayground(email, workspaceID, df,queryColName=queryColName, dataStreamName=streamId):
                 print(
                     "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
                 )

llumo/helpingFuntions.py

@@ -7,6 +7,7 @@ import requests
 import json
 import base64
 import os
+import re
 
 subscriptionUrl = "https://app.llumo.ai/api/workspace/record-extra-usage"
 getStreamdataUrl = "https://app.llumo.ai/api/data-stream/all"
@@ -18,6 +19,8 @@ uploadColList = (
 uploadRowList = (
     "https://app.llumo.ai/api/New-Eval-API/new-upload-flow/uploadRowsInDBPlayground"
 )
+createInsightUrl = "https://app.llumo.ai/api/New-Eval-API/insights-api/generate-playground-insights"
+getPlaygroundInsightsUrl="https://app.llumo.ai/api/New-Eval-API/insights-api/get-all-playground-insights"
 
 
 def getProcessID():
@@ -210,30 +213,151 @@ def deleteColumnListInPlayground(workspaceID: str, playgroundID: str):
         return None
 
 
-def createColumn(workspaceID, dataframe, playgroundID):
-
+def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColName=None,outputColName= "output",dataStreamName=None,definationMapping=None):
     if len(dataframe) > 100:
         dataframe = dataframe.head(100)
         print("⚠️ Dataframe truncated to 100 rows for upload.")
 
-    playgroundID = playgroundID
-
     coltemplate = {
         "workspaceID": workspaceID,
         "playgroundID": playgroundID,
         "columnListToUpload": [],
     }
+    allEvals = ['Response Completeness', 'Response Bias', 'Response Harmfulness', 'Input Toxicity', 'Input Harmfulness', 'Context Utilization', 'Relevance Retention', 'Semantic Cohesion', 'Final Task Alignment', 'Tool Reliability', 'Response Correctness', 'Response Toxicity', 'Input Bias', 'Input Relevancy', 'Redundancy Reduction', 'Response Sentiment', 'Tool Selection Accuracy', 'Stepwise Progression', 'Hallucination', 'Faithfulness', 'Answer Relevancy', 'Context Precision', 'Answer Similarity', 'Harmfulness', 'Maliciousness', 'Coherence', 'Answer Correctness', 'Context Recall', 'Context Entity Recall', 'Conciseness', 'customEvalColumn', 'Groundedness', 'Memory Utilization', 'Input Relevancy (Multi-turn)']
+
+
+
+    # Create a mapping of column names to unique column IDs
+    columnIDMapping = {}
 
+    # Iterate over each column in the dataframe
     for indx, col in enumerate(dataframe.columns):
-        template = {
-            "label": col,
-            "type": "VARIABLE",
-            "variableType": "STRING",
-            "order": indx,
-            "columnID": col,
-        }
+        # Generate a unique column ID using uuid
+        columnID = str(uuid.uuid4().hex[:8]) 
+
+        columnIDMapping[col] = columnID
+
+        
+        if col.startswith('output'):
+            # For output columns, create the prompt template with promptText
+            if promptText:
+                # Extract variables from promptText and set them as dependencies
+                dependencies = []
+
+                # Find variables inside {{variable}}
+                variables = re.findall(r'{{(.*?)}}', promptText)
+
+                # Loop through each variable and check if it exists as a column name
+                for var in variables:
+                    varName = var.strip() 
+                    if varName in columnIDMapping:  # Check if the variable is a column name
+                        dependencies.append(columnIDMapping[varName])  # Add its columnID
+
+                # Now update the template for the output column
+                
+                template={
+                    "provider": "OPENAI",
+                    "model": "GPT_4o",
+                    "promptText": promptText,
+                    "modelOptions": {
+                        "temperature": 0,
+                        "frequencyPenalty": 0,
+                        "presencePenalty": 0,
+                        "maxToken": 8192
+                    },
+                    "toolConfig": "none",
+                    "concurrency": "",
+                    "outputType": "STRING",
+                    "isPromptSelected": True,
+                    "isSmartPromptSelected": False,
+                    "dependency": dependencies,  # Use the dependencies extracted from promptText
+                    "columnID": columnID,  # Use the generated column ID
+                    "label": col,
+                    "type": "PROMPT",
+                    "order": indx,
+                }
+                
+        elif col.startswith('Data '):
+            if queryColName and dataStreamName:
+                dependencies = []
+                dependencies.append(columnIDMapping[queryColName])
+                template = {
+                    "variableType": "STRING",
+                    "dependency": dependencies,
+                    "dataStreamName": dataStreamName,
+                    "query": columnIDMapping[queryColName],
+                    "columnID": columnID,  # Use the generated column ID
+                    "label": "Data stream",
+                    "type": "DATA_STREAM",
+                    "order": indx}
+
+        elif col in allEvals:
+
+            dependencies = []
+            variables = re.findall(r'{{(.*?)}}', promptText)
+
+                # Loop through each variable and check if it exists as a column name
+            for var in variables:
+                varName = var.strip() 
+                if varName in columnIDMapping:  # Check if the variable is a column name
+                    dependencies.append(columnIDMapping[varName])
+                    
+            dependencies.append(columnIDMapping[outputColName])  # Add the output column ID
+            longDef = definationMapping.get(col, {}).get(col, "")
+            shortDef ="You have run this from SDK"
+            enum =  col.upper().replace(" ","_")
+
+            template = {
+      "analytics": [
+        col.lower().replace(" ","_")
+      ],
+      "evaluationMetric": "ALL",
+      "evaluationModel": "GEMINI_PRO",
+      "selectPrompt": columnIDMapping[outputColName],
+      "scoreCondition": "GREATER_THAN",
+      "scoreValue": "50",
+      "scoreResult": "PASS",
+      "llmKpi": col,
+      "setRules": True,
+      "type": "EVAL",
+      "evalType": "LLM",
+      "similarityMetric": None,
+      "embeddingModel": None,
+      "groundTruth": None,
+      "dataStream": None,
+      "context": None,
+      "dependency": [columnIDMapping[outputColName]],
+      "hallucinationFields": {
+        "query": None,
+        "context": None,
+        "output": None
+      },
+      "definition": longDef,
+      "analyticsENUM": enum,
+      "prompt": shortDef,
+      "analyticsName": col,
+      "columnID": str(uuid.uuid4().hex[:8]),
+      "label": col,
+      "order": indx
+    }
+
+           
+
+            
+        else:
+            
+            template = {
+                "label": col,  # Label is the column name
+                "type": "VARIABLE",  # Default type for non-output columns
+                "variableType": "STRING",
+                "order": indx,
+                "columnID": columnID,  # Use the generated column ID
+            }
+
+        # Append the template to the column list
         coltemplate["columnListToUpload"].append(template)
 
+    # Prepare the row template structure
     rowTemplate = {
         "workspaceID": workspaceID,
         "playgroundID": playgroundID,
@@ -241,14 +365,22 @@ def createColumn(workspaceID, dataframe, playgroundID):
         "columnList": coltemplate["columnListToUpload"],
     }
 
+    # Populate dataToUploadList with rows from the dataframe
     for indx, row in dataframe.iterrows():
-        row_dict = row.to_dict()
+        row_dict = {}
+
+        # For each column, we need to map the column ID to the corresponding value in the row
+        for col in dataframe.columns:
+            columnID = columnIDMapping[col]  # Get the columnID from the mapping
+            row_dict[columnID] = row[col]  # Map the columnID to the value in the row
+
+        # Add the row index (if necessary)
         row_dict["pIndex"] = indx
         rowTemplate["dataToUploadList"].append(row_dict)
 
+    # Return the column template, row template, and the column ID mapping
     return coltemplate, rowTemplate
 
-
 def uploadColumnListInPlayground(payload):
     url = uploadColList
     headers = {
@@ -292,10 +424,11 @@ def uploadRowsInDBPlayground(payload):
         return None
 
 
-def createPlayground(email, workspaceID, df):
+def createPlayground(email, workspaceID, df, promptText=None,queryColName=None,dataStreamName=None,definationMapping=None,outputColName="output"):
+    
     playgroundId = str(createEvalPlayground(email=email, workspaceID=workspaceID))
     payload1, payload2 = createColumn(
-        workspaceID=workspaceID, dataframe=df, playgroundID=playgroundId
+        workspaceID=workspaceID, dataframe=df, playgroundID=playgroundId, promptText=promptText,queryColName=queryColName,dataStreamName=dataStreamName,definationMapping=definationMapping,outputColName=outputColName
     )
     deleteExistingRows = deleteColumnListInPlayground(
         workspaceID=workspaceID, playgroundID=playgroundId
@@ -305,3 +438,56 @@ def createPlayground(email, workspaceID, df):
 
     if rowListUpload:
         return True
+
+
+
+def getPlaygroundInsights(workspaceID: str, activePlayground: str):
+    headers = {
+        
+        "Content-Type": "application/json",
+    }
+
+    # Initial request to generate playground insights
+    payload = {
+        "activePlayground": activePlayground,
+        "workspaceID": workspaceID,
+    }
+
+    urlGenerate = createInsightUrl
+    
+    responseGenerate = requests.post(urlGenerate, json=payload, headers=headers)
+
+    if responseGenerate.status_code == 200:
+        responseJson = responseGenerate.json()
+        
+        insightStatus = responseJson.get("data", {}).get("insight", False)
+
+        if insightStatus:
+            # If insight is true, request to get all playground insights
+            urlGetAll = getPlaygroundInsightsUrl
+            
+            responseGetAll = requests.post(urlGetAll, json=payload, headers=headers)
+
+            if responseGetAll.status_code == 200:
+                responseAllJson = responseGetAll.json()
+                
+                data = responseAllJson.get("data", {}).get("data", [])
+                
+                # Extract insight and solution
+                insights = []
+                for item in data:
+                    insight = item.get("insight", "")
+                    solution = item.get("solution", "")
+                    insights.append({"insight": insight, "solution": solution})
+
+                return insights
+            else:
+                print(f"Error fetching all insights: {responseGetAll.status_code} - {responseGetAll.text}")
+                return None
+        else:
+            print("No insight generated.")
+            return None
+    else:
+        print(f"Error generating insight: {responseGenerate.status_code} - {responseGenerate.text}")
+        return None
+

llumo/sockets.py

@@ -128,7 +128,7 @@ class LlumoSocketClient:
                     self._listening_done.set()
                     break
 
-                time.sleep(1)
+                time.sleep(3)
 
         timeout_thread = threading.Thread(target=timeout_watcher, daemon=True)
         timeout_thread.start()

{llumo-0.2.12.dist-info → llumo-0.2.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.12
+Version: 0.2.13
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

llumo-0.2.13.dist-info/RECORD

@@ -0,0 +1,13 @@
+llumo/__init__.py,sha256=O04b4yW1BnOvcHzxWFddAKhtdBEhBNhLdb6xgnpHH_Q,205
+llumo/client.py,sha256=mVqxNyYMbHlL9lnkpw_H6_n98WYO38ZcLZNnQX1TjUE,35948
+llumo/exceptions.py,sha256=iCj7HhtO_ckC2EaVBdXbAudNpuMDsYmmMEV5lwynZ-E,1854
+llumo/execution.py,sha256=x88wQV8eL99wNN5YtjFaAMCIfN1PdfQVlAZQb4vzgQ0,1413
+llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
+llumo/helpingFuntions.py,sha256=ah0FUQcRV3gfguvjQQ_aZzq59hpJttqAPJdjJVNYdFc,17110
+llumo/models.py,sha256=YH-qAMnShmUpmKE2LQAzQdpRsaXkFSlOqMxHwU4zBUI,1560
+llumo/sockets.py,sha256=i90l2rr08paa-ifKy2E5YMIS8r3yRBmu2CUOjhFKork,5579
+llumo-0.2.13.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
+llumo-0.2.13.dist-info/METADATA,sha256=W-ZVsTKhiPw6strlNQEqyufgOLfV9ZCjCpY6Dj3Qc94,1491
+llumo-0.2.13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+llumo-0.2.13.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
+llumo-0.2.13.dist-info/RECORD,,

llumo-0.2.12.dist-info/RECORD

@@ -1,13 +0,0 @@
-llumo/__init__.py,sha256=O04b4yW1BnOvcHzxWFddAKhtdBEhBNhLdb6xgnpHH_Q,205
-llumo/client.py,sha256=QMasCjzRmL38RuMUHoY39_ge2nIRdSOECjj1daicd-k,34144
-llumo/exceptions.py,sha256=iCj7HhtO_ckC2EaVBdXbAudNpuMDsYmmMEV5lwynZ-E,1854
-llumo/execution.py,sha256=x88wQV8eL99wNN5YtjFaAMCIfN1PdfQVlAZQb4vzgQ0,1413
-llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
-llumo/helpingFuntions.py,sha256=lG_d3lQgJj6pI7v1YdLqdPojrLCNwybKz29zXrGaL5k,9090
-llumo/models.py,sha256=YH-qAMnShmUpmKE2LQAzQdpRsaXkFSlOqMxHwU4zBUI,1560
-llumo/sockets.py,sha256=Qxxqtx3Hg07HLhA4QfcipK1ChiOYhHZBu02iA6MfYlQ,5579
-llumo-0.2.12.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
-llumo-0.2.12.dist-info/METADATA,sha256=CYu8qMtD-XZuoQzC7YPZRlNgetEQPjMT1RiMGnHdZJs,1491
-llumo-0.2.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-llumo-0.2.12.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
-llumo-0.2.12.dist-info/RECORD,,