llumo 0.2.15b2__py3-none-any.whl → 0.2.16b1__py3-none-any.whl

llumo/__init__.py

@@ -4,4 +4,5 @@ from .helpingFuntions import *
 from .models import AVAILABLEMODELS
 from .execution import ModelExecutor
 from .functionCalling import *
-
+from .openai import openai
+from .google import genai

llumo/chains.py

@@ -0,0 +1,73 @@
+import pandas as pd
+from .helpingFuntions import *
+
+class LlumoDataFrameResults(pd.DataFrame):
+    _metadata=["evals","evalData","definationMapping"]
+
+    def __init__(self, *args,evals=None,evalData=None,definationMapping=None,**kwargs):
+        self.evals = evals or []
+        self.evalData= evalData or []
+        self.definationMapping= definationMapping or {}
+        super().__init__(*args, **kwargs)
+
+    @property
+    def _constructor(self):
+        # Needed so slicing operations return the same type
+        return LlumoDataFrameResults
+
+    def insights(self):
+        
+        if not self.evalData:
+            print("No raw data available. Please run evaluateMultiple() first.")
+            return None
+        try:
+            insights=[]
+            reasonData,uniqueEdgecase=groupLogsByClass(self.evalData) # print(rawResults)
+            
+            for evalname in self.evals:
+                uniqueclassesstring = ",".join(uniqueEdgecase.get(evalname, []))
+                allReasons = []
+                for edgeCase in reasonData[evalname]:
+                    allReasons.extend(reasonData[evalname][edgeCase])
+                
+                evalDefinition = self.definationMapping.get(evalname, {}).get("definition", "")
+
+                insights.append(getPlaygroundInsights(evalDefinition,uniqueclassesstring,allReasons))
+            return insights
+        except Exception as e:
+
+            print("Can not genrate insights for this eval, please try again later.")
+
+
+class LlumoDictResults(list):
+    _metadata=["evals","evalData","definationMapping"]
+
+    def __init__(self, *args,evals=None,evalData=None,definationMapping=None,**kwargs):
+        self.evals = evals or []
+        self.evalData= evalData or []
+        self.definationMapping= definationMapping or {}
+        super().__init__(*args, **kwargs) # This will handle list[dict]
+
+    def insights(self):
+        
+        if not self.evalData:
+            print("No raw data available. Please run evaluateMultiple() first.")
+            return None
+        try:
+            insights=[]
+            reasonData,uniqueEdgecase=groupLogsByClass(self.evalData) # print(rawResults)
+            for evalname in self.evals:
+                uniqueclassesstring = ",".join(uniqueEdgecase.get(evalname, []))
+                allReasons = []
+                for edgeCase in reasonData[evalname]:
+                    allReasons.extend(reasonData[evalname][edgeCase])
+                evalDefinition = self.definationMapping.get(evalname, {}).get("definition", "")
+                insights.append(getPlaygroundInsights(evalDefinition,uniqueclassesstring,allReasons))
+            return insights
+        except Exception as e:
+            print("Can not genrate insights for this eval, please try again later.")
+    
+    
+for _cls in (LlumoDataFrameResults, LlumoDictResults):
+    _cls.__name__ = "LlumoResults"
+    _cls.__qualname__ = "LlumoResults"

llumo/client.py

@@ -5,7 +5,7 @@ import time
 import re
 import json
 import uuid
-
+import warnings
 import os
 import itertools
 import pandas as pd
@@ -16,9 +16,12 @@ from .exceptions import LlumoAIError
 from .helpingFuntions import *
 from .sockets import LlumoSocketClient
 from .functionCalling import LlumoAgentExecutor
+from .chains import LlumoDataFrameResults,LlumoDictResults
 import threading
 from tqdm import tqdm
 
+pd.set_option('future.no_silent_downcasting', True)
+
 postUrl = (
     "https://red-skull-service-392377961931.us-central1.run.app/api/process-playground"
 )
@@ -38,7 +41,8 @@ class LlumoClient:
 
     def __init__(self, api_key):
         self.apiKey = api_key
-        
+        self.evalData=[]
+        self.evals=[]
         self.processMapping = {}
         self.definationMapping = {}
 
@@ -393,27 +397,46 @@ class LlumoClient:
                 outputColName=outputColName,
             ):
                 print(
-                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results.Please rerun the experiment to see the results on playground."
+                    "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results.Please rerun the experiment to see the results on playground."
                 )
         else:
             return dataframe
 
     # this function allows the users to run multiple evals at once
 
-    def evaluateCompressor(self, data, prompt_template):
+    def compressor(self, data, prompt_template):
         results = []
         dataframe = pd.DataFrame(data)
+
         try:
-            socketID = self.socket.connect(timeout=150)
+            self.socket = LlumoSocketClient(socketUrl)
+            dataframe = pd.DataFrame(data).astype(str)
+            socketID = self.socket.connect(timeout=250)
+
+            # Wait for socket connection
             max_wait_secs = 20
             waited_secs = 0
             while not self.socket._connection_established.is_set():
                 time.sleep(0.1)
                 waited_secs += 0.1
                 if waited_secs >= max_wait_secs:
-                    raise RuntimeError(
-                        "Timeout waiting for server 'connection-established' event."
-                    )
+                    raise RuntimeError("Timeout waiting for server connection")
+
+            # Start listener thread
+            expectedResults = len(dataframe)
+            # print("expected result" ,expectedResults)
+            timeout = max(100, min(150, expectedResults * 10))
+            listener_thread = threading.Thread(
+                target=self.socket.listenForResults,
+                kwargs={
+                    "min_wait": 40,
+                    "max_wait": timeout,
+                    "inactivity_timeout": 10,
+                    "expected_results": expectedResults,
+                },
+                daemon=True,
+            )
+            listener_thread.start()
 
             try:
                 self.validateApiKey()
@@ -508,7 +531,7 @@ class LlumoClient:
                     "playgroundID": activePlayground,
                 }
 
-                rowIdMapping[rowID] = index
+                rowIdMapping[f'{rowID}-{columnID}-{columnID}'] = index
                 # print("__________________________TEMPLATE__________________________________")
                 # print(templateData)
 
@@ -537,7 +560,19 @@ class LlumoClient:
                 expected_results=None,
             )
 
-            results = self.socket.getReceivedData()
+            rawResults = self.socket.getReceivedData()
+            receivedRowIDs = {key for item in rawResults for key in item.keys()}
+            expectedRowIDs = set(rowIdMapping.keys())
+            missingRowIDs = expectedRowIDs - receivedRowIDs
+            # print("All expected keys:", expected_rowIDs)
+            # print("All received keys:", received_rowIDs)
+            # print("Missing keys:", len(missingRowIDs))
+            missingRowIDs = list(missingRowIDs)
+
+            if len(missingRowIDs) > 0:
+                dataFromDb = fetchData(workspaceID, activePlayground, missingRowIDs)
+                rawResults.extend(dataFromDb)
+
             # results = self.finalResp(eval_results)
             # print(f"======= Completed evaluation: {eval} =======\n")
 
@@ -551,10 +586,10 @@ class LlumoClient:
                 print(f"Error disconnecting socket: {e}")
 
         dataframe["Compressed Input"] = None
-        for records in results:
+        for records in rawResults:
             for compound_key, value in records.items():
                 # for compound_key, value in item['data'].items():
-                rowID = compound_key.split("-")[0]
+                rowID = compound_key
                 # looking for the index of each rowID , in the original dataframe
                 if rowID in rowIdMapping:
                     index = rowIdMapping[rowID]
@@ -580,6 +615,7 @@ class LlumoClient:
         prompt_template="",
         outputColName="output",
         createExperiment: bool = False,
+        getDataFrame:bool =False,
         _tocheck=True,
     ):
         self.socket = LlumoSocketClient(socketUrl)
@@ -587,6 +623,8 @@ class LlumoClient:
         workspaceID = None
         email = None
         socketID = self.socket.connect(timeout=250)
+        self.evalData=[]
+        self.evals=evals
         self.allBatches = []
         rowIdMapping = {}  # (rowID-columnID-columnID -> (index, evalName))
 
@@ -614,10 +652,14 @@ class LlumoClient:
             daemon=True,
         )
         listener_thread.start()
-        
-        activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
-                    "-", ""
-                )        
+        self.validateApiKey(evalName=evals[0])
+        if createExperiment:
+            activePlayground = str(createEvalPlayground(email=self.email, workspaceID=self.workspaceID))
+            
+        else:    
+            activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
+                        "-", ""
+                    )        
         for evalName in evals:
             # print(f"\n======= Running evaluation for: {evalName} =======")
 
@@ -776,10 +818,8 @@ class LlumoClient:
             rawResults.extend(dataFromDb)
             
             
+        self.evalData = rawResults
             
-            
-            
-
         # Initialize dataframe columns for each eval
         for eval in evals:
             dataframe[eval] = None
@@ -799,7 +839,12 @@ class LlumoClient:
 
         if createExperiment:
             pd.set_option("future.no_silent_downcasting", True)
-            df = dataframe.fillna("Some error occured").astype(object)
+            # df = dataframe.fillna("Some error occured").astype(object)
+            with warnings.catch_warnings():
+                warnings.simplefilter(action='ignore', category=FutureWarning)
+                df = dataframe.fillna("Some error occurred").astype(str)
+
+            df = dataframe.fillna("Some error occured").infer_objects(copy=False)
             if createPlayground(
                 email,
                 workspaceID,
@@ -807,13 +852,19 @@ class LlumoClient:
                 promptText=prompt_template,
                 definationMapping=self.definationMapping,
                 outputColName=outputColName,
+                activePlayground= activePlayground
             ):
                 print(
-                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
+                    "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results."
                 )
         else:
-            return dataframe
+            if getDataFrame:
+                return LlumoDataFrameResults(dataframe,evals=self.evals,evalData=self.evalData,definationMapping=self.definationMapping)
+            else:
+                data=dataframe.to_dict(orient="records")
+                return LlumoDictResults(data,evals=self.evals,evalData=self.evalData,definationMapping=self.definationMapping)
 
+    
     def promptSweep(
         self,
         templates: List[str],
@@ -823,6 +874,7 @@ class LlumoClient:
         evals=["Response Correctness"],
         toEvaluate: bool = False,
         createExperiment: bool = False,
+        getDataFrame = False
 
 
     ) -> pd.DataFrame:
@@ -878,6 +930,8 @@ class LlumoClient:
                         prompt_template=str(templates[0]),
                         outputColName=outputColName,
                         _tocheck=False,
+                        getDataFrame=True,
+                        createExperiment = False
                     )
 
                     # Rename all new columns with _i+1 (e.g., _1, _2)
@@ -910,10 +964,17 @@ class LlumoClient:
                 ):
 
                     print(
-                        "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
+                        "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results."
                     )
             else:
-                return dfWithEvals
+                if getDataFrame:
+                    return LlumoDataFrameResults(dfWithEvals, evals=self.evals, evalData=self.evalData,
+                                                 definationMapping=self.definationMapping)
+                else:
+                    data = dfWithEvals.to_dict(orient="records")
+                    return LlumoDictResults(data, evals=self.evals, evalData=self.evalData,definationMapping=self.definationMapping)
+
+
         else:
             if createExperiment == True:
                 pd.set_option("future.no_silent_downcasting", True)
@@ -921,10 +982,18 @@ class LlumoClient:
 
                 if createPlayground(email, workspaceID, df, promptText=templates[0]):
                     print(
-                        "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
+                        "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results."
                     )
             else:
-                return df
+                if getDataFrame:
+                    return LlumoDataFrameResults(df, evals=self.evals, evalData=self.evalData,
+                                                 definationMapping=self.definationMapping)
+                else:
+                    data = df.to_dict(orient="records")
+                    return LlumoDictResults(data, evals=self.evals, evalData=self.evalData,
+                                            definationMapping=self.definationMapping)
+
+
 
     # this function generates an output using llm and tools and evaluate that output
     def evaluateAgents(
@@ -936,6 +1005,7 @@ class LlumoClient:
         evals=["Final Task Alignment"],
         prompt_template="Give answer for the given query: {{query}}",
         createExperiment: bool = False,
+        getDataFrame:bool = False
 
     ):
         if model.lower() not in ["openai", "google"]:
@@ -961,27 +1031,33 @@ class LlumoClient:
             toolResponseDf.to_dict(orient="records"),
             evals=evals,
             prompt_template=prompt_template,
-            createExperiment=False,
+            createExperiment=createExperiment,
+            getDataFrame=getDataFrame
+
         )
 
-        if createExperiment:
-            pd.set_option("future.no_silent_downcasting", True)
-            df = toolResponseDf.fillna("Some error occured")
-            if createPlayground(self.email, self.workspaceID, df,promptText=prompt_template,definationMapping=self.definationMapping):
-                print(
-                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
-                )
-        else:
-            return toolResponseDf
+        return toolResponseDf
+        # if createExperiment:
+        #     pd.set_option("future.no_silent_downcasting", True)
+        #     df = toolResponseDf.fillna("Some error occured")
+        #     if createPlayground(self.email, self.workspaceID, df,promptText=prompt_template,definationMapping=self.definationMapping):
+        #         print(
+        #             "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results."
+        #         )
+        # else:
+        #     return toolResponseDf
 
     # this function evaluate that tools output given by the user
     def evaluateAgentResponses(
         self,
         data,
         evals=["Final Task Alignment"],
-        outputColName="output",
         createExperiment: bool = False,
+        getDataFrame = False,
+        outputColName="output"
+
     ):
+
         dataframe = pd.DataFrame(data)
 
         try:
@@ -1001,7 +1077,9 @@ class LlumoClient:
                 evals=evals,
                 prompt_template="Give answer for the given query: {{query}}",
                 outputColName=outputColName,
-                createExperiment=createExperiment
+                createExperiment=createExperiment,
+                getDataFrame = getDataFrame
+
             )
             if createExperiment:
                 pass
@@ -1022,7 +1100,8 @@ class LlumoClient:
             prompt_template="Give answer to the given: {{query}} using the context:{{context}}",
             evals=["Context Utilization"],
             toEvaluate=False,
-            generateOutput=True
+            generateOutput=True,
+            getDataFrame = False
     ):
         # Validate required parameters
         if generateOutput:
@@ -1147,7 +1226,6 @@ class LlumoClient:
         outputEvalMapping = None
         if toEvaluate:
             for evalName in evals:
-
                 # Validate API and dependencies
                 self.validateApiKey(evalName=evalName)
                 metricDependencies = checkDependency(
@@ -1156,14 +1234,17 @@ class LlumoClient:
                 if not metricDependencies["status"]:
                     raise LlumoAIError.dependencyError(metricDependencies["message"])
 
-            working_df, outputEvalMapping = self._evaluateForStream(working_df, evals, modelAliases, prompt_template)
-
+            working_df, outputEvalMapping = self._evaluateForStream(working_df, evals, modelAliases, prompt_template,generateOutput)
 
+        
         self.socket.disconnect()
 
         # Create experiment if required
         if createExperiment:
-            df = working_df.fillna("Some error occured").astype(object)
+            # df = working_df.fillna("Some error occured").astype(object)
+            with warnings.catch_warnings():
+                warnings.simplefilter(action='ignore', category=FutureWarning)
+                df = working_df.fillna("Some error occurred").astype(str)
             if createPlayground(
                     email, workspaceID, df,
                     queryColName=queryColName,
@@ -1173,10 +1254,22 @@ class LlumoClient:
                     evalOutputMap=outputEvalMapping
             ):
                 print(
-                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results.")
+                    "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results.")
         else:
-            self.latestDataframe = working_df
-            return working_df
+            if getDataFrame == True and toEvaluate == True:
+                return LlumoDataFrameResults(working_df, evals=self.evals, evalData=self.evalData,
+                                             definationMapping=self.definationMapping)
+
+            elif getDataFrame == False and toEvaluate == True:
+                data = working_df.to_dict(orient="records")
+                return LlumoDictResults(data, evals=self.evals, evalData=self.evalData,
+                                        definationMapping=self.definationMapping)
+
+            elif getDataFrame== True and toEvaluate == False:
+                return working_df
+
+            elif getDataFrame == False and toEvaluate == False :
+                return working_df.to_dict(orient = "records")
 
     def _outputForStream(self, df, modelAliases, prompt_template, apiKey):
         executor = ModelExecutor(apiKey)
@@ -1192,50 +1285,78 @@ class LlumoClient:
 
                     provider = getProviderFromModel(model)
                     if provider == Provider.OPENAI:
-                        print(validateOpenaiKey(apiKey))
+                        validateOpenaiKey(apiKey)
                     elif provider == Provider.GOOGLE:
                         validateGoogleKey(apiKey)
 
                     filled_template = getInputPopulatedPrompt(prompt_template, inputDict)
                     response = executor.execute(provider, model.value, filled_template, apiKey)
                     df.at[indx, f"output_{i}"] = response
+                    
                 except Exception as e:
                     # df.at[indx, f"output_{i}"] = str(e)
                     raise e
 
         return df
 
-    def _evaluateForStream(self, df, evals, modelAliases, prompt_template):
+    def _evaluateForStream(self, df, evals, modelAliases, prompt_template, generateOutput):
         dfWithEvals = df.copy()
-
         outputColMapping = {}
 
-        for i, model in enumerate(modelAliases, 1):
-            outputColName = f"output_{i}"
-            try:
+        if generateOutput:
+            # Evaluate per model output
+            for i, model in enumerate(modelAliases, 1):
+                outputColName = f"output_{i}"
+                try:
+                    res = self.evaluateMultiple(
+                        dfWithEvals.to_dict("records"),
+                        evals=evals,
+                        prompt_template=prompt_template,
+                        outputColName=outputColName,
+                        _tocheck=False,
+                        getDataFrame=True,
+                        createExperiment=False
+                    )
 
+                    for evalMetric in evals:
+                        scoreCol = f"{evalMetric}"
+                        reasonCol = f"{evalMetric} Reason"
+                        if scoreCol in res.columns:
+                            res = res.rename(columns={scoreCol: f"{scoreCol}_{i}"})
+                        if reasonCol in res.columns:
+                            res = res.rename(columns={reasonCol: f"{evalMetric}_{i} Reason"})
+
+                        outputColMapping[f"{scoreCol}_{i}"] = outputColName
+
+                    newCols = [col for col in res.columns if col not in dfWithEvals.columns]
+                    dfWithEvals = pd.concat([dfWithEvals, res[newCols]], axis=1)
+
+                except Exception as e:
+                    print(f"Evaluation failed for model {model.value}: {str(e)}")
+
+        else:
+            # Evaluate only once on "output" column
+            try:
+                outputColName = "output"
                 res = self.evaluateMultiple(
                     dfWithEvals.to_dict("records"),
                     evals=evals,
                     prompt_template=prompt_template,
                     outputColName=outputColName,
                     _tocheck=False,
+                    getDataFrame=True,
+                    createExperiment=False
                 )
                 for evalMetric in evals:
                     scoreCol = f"{evalMetric}"
                     reasonCol = f"{evalMetric} Reason"
-
-                    if scoreCol in res.columns:
-                        res = res.rename(columns={scoreCol: f"{scoreCol}_{i}"})
-                    if reasonCol in res.columns:
-                        res = res.rename(columns={reasonCol: f"{evalMetric}_{i} Reason"})
-
-                    outputColMapping[f"{scoreCol}_{i}"] = outputColName
+                    outputColMapping[scoreCol] = "output"
 
                 newCols = [col for col in res.columns if col not in dfWithEvals.columns]
                 dfWithEvals = pd.concat([dfWithEvals, res[newCols]], axis=1)
             except Exception as e:
-                print(f"Evaluation failed for model {model.value}: {str(e)}")
+                print(f"Evaluation failed: {str(e)}")
+
         return dfWithEvals, outputColMapping
 
     def runDataStream(
@@ -1244,6 +1365,7 @@ class LlumoClient:
             streamName: str,
             queryColName: str = "query",
             createExperiment: bool = False,
+            getDataFrame = False
            ):
 
 
@@ -1362,10 +1484,16 @@ class LlumoClient:
                     definationMapping=self.definationMapping,
             ):
                 print(
-                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results.")
+                    "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results.")
         else:
-            self.latestDataframe = working_df
-            return working_df
+            if getDataFrame:
+                return working_df
+
+            else:
+                data = working_df.to_dict(orient="records")
+                return data
+            # self.latestDataframe = working_df
+            # return working_df
 
 
     def createExperiment(self, dataframe):
@@ -1375,7 +1503,7 @@ class LlumoClient:
             flag = createPlayground(self.email, self.workspaceID, dataframe)
             if flag:
                 print(
-                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
+                    "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results."
                 )
         except Exception as e:
             raise "Some error ocuured please check your API key"
@@ -1415,7 +1543,7 @@ class LlumoClient:
             if createPlayground(self.email, self.workspaceID, df):
                 
                 print(
-                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
+                    "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results."
                 )
                 
                 return True

llumo/google.py

@@ -0,0 +1,34 @@
+from google import generativeai as _genai
+
+class genai:
+    """
+    Top-level wrapper module to mimic:
+    >>> from google import genai
+    >>> client = genai.Client(api_key=...)
+    """
+
+    class Client:
+        def __init__(self, api_key: str, default_model: str = "gemini-2.5-flash"):
+            _genai.configure(api_key=api_key)
+            self._defaultModel = default_model
+            self._defaultModelInstance = _genai.GenerativeModel(model_name=default_model)
+
+            class Models:
+                def __init__(self, outer):
+                    self._outer = outer
+
+                def generate_content(self, contents: str | list[str], model: str = None, **kwargs):
+                    model_name = model or self._outer._defaultModel
+                    model_instance = _genai.GenerativeModel(model_name=model_name)
+                    return model_instance.generate_content(contents=contents, **kwargs)
+
+            self.models = Models(self)
+
+        def generate(self, prompt: str | list[str], **kwargs):
+            """Convenience shortcut for single-line generation."""
+            return self._defaultModelInstance.generate_content(prompt, **kwargs)
+
+        def setDefaultModel(self, model_name: str):
+            """Change the default model at runtime."""
+            self._defaultModel = model_name
+            self._defaultModelInstance = _genai.GenerativeModel(model_name=model_name)

llumo/helpingFuntions.py

@@ -10,6 +10,7 @@ import os
 import re
 import openai
 import google.generativeai as genai
+from collections import defaultdict
 
 
 from .models import  _MODEL_METADATA,  AVAILABLEMODELS
@@ -23,8 +24,7 @@ uploadColList = (
 uploadRowList = (
     "https://app.llumo.ai/api/New-Eval-API/new-upload-flow/uploadRowsInDBPlayground"
 )
-createInsightUrl = "https://app.llumo.ai/api/New-Eval-API/insights-api/generate-playground-insights"
-getPlaygroundInsightsUrl="https://app.llumo.ai/api/New-Eval-API/insights-api/get-all-playground-insights"
+createInsightUrl="https://app.llumo.ai/api/external/generate-insight-from-eval-for-sdk"
 
 
 def getProcessID():
@@ -229,11 +229,12 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
     }
     allEvals = ['Response Completeness', 'Response Bias', 'Response Harmfulness', 'Input Toxicity', 'Input Harmfulness', 'Context Utilization', 'Relevance Retention', 'Semantic Cohesion', 'Final Task Alignment', 'Tool Reliability', 'Response Correctness', 'Response Toxicity', 'Input Bias', 'Input Relevancy', 'Redundancy Reduction', 'Response Sentiment', 'Tool Selection Accuracy', 'Stepwise Progression', 'Hallucination', 'Faithfulness', 'Answer Relevancy', 'Context Precision', 'Answer Similarity', 'Harmfulness', 'Maliciousness', 'Coherence', 'Answer Correctness', 'Context Recall', 'Context Entity Recall', 'Conciseness', 'customEvalColumn', 'Groundedness', 'Memory Utilization', 'Input Relevancy (Multi-turn)']
 
-
+    evalDependencies =  checkDependency(_returnDepMapping=True)
 
     # Create a mapping of column names to unique column IDs
     columnIDMapping = {}
-
+    print("Def Mapping: ")
+    print(definationMapping)
     # Iterate over each column in the dataframe
     for indx, col in enumerate(dataframe.columns):
         # Generate a unique column ID using uuid
@@ -242,46 +243,46 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
         columnIDMapping[col] = columnID
 
 
-        if col.startswith('output') and promptText!=None:
-            # For output columns, create the prompt template with promptText
-            if promptText:
-                # Extract variables from promptText and set them as dependencies
-                dependencies = []
-
-                # Find variables inside {{variable}}
-                variables = re.findall(r'{{(.*?)}}', promptText)
-
-                # Loop through each variable and check if it exists as a column name
-                for var in variables:
-                    varName = var.strip()
-                    if varName in columnIDMapping:  # Check if the variable is a column name
-                        dependencies.append(columnIDMapping[varName])  # Add its columnID
-
-                # Now update the template for the output column
-
-                template={
-                    "provider": "OPENAI",
-                    "model": "GPT_4o",
-                    "promptText": promptText,
-                    "modelOptions": {
-                        "temperature": 0,
-                        "frequencyPenalty": 0,
-                        "presencePenalty": 0,
-                        "maxToken": 8192
-                    },
-                    "toolConfig": "none",
-                    "concurrency": "",
-                    "outputType": "STRING",
-                    "isPromptSelected": True,
-                    "isSmartPromptSelected": False,
-                    "dependency": dependencies,  # Use the dependencies extracted from promptText
-                    "columnID": columnID,  # Use the generated column ID
-                    "label": col,
-                    "type": "PROMPT",
-                    "order": indx,
-                }
-
-        elif col.startswith('context') and dataStreamName != None :
+        # if col.startswith('output') and promptText!=None:
+        #     # For output columns, create the prompt template with promptText
+        #     if promptText:
+        #         # Extract variables from promptText and set them as dependencies
+        #         dependencies = []
+        #
+        #         # Find variables inside {{variable}}
+        #         variables = re.findall(r'{{(.*?)}}', promptText)
+        #
+        #         # Loop through each variable and check if it exists as a column name
+        #         for var in variables:
+        #             varName = var.strip()
+        #             if varName in columnIDMapping:  # Check if the variable is a column name
+        #                 dependencies.append(columnIDMapping[varName])  # Add its columnID
+        #
+        #         # Now update the template for the output column
+        #
+        #         template={
+        #             "provider": "OPENAI",
+        #             "model": "GPT_4o",
+        #             "promptText": promptText,
+        #             "modelOptions": {
+        #                 "temperature": 0,
+        #                 "frequencyPenalty": 0,
+        #                 "presencePenalty": 0,
+        #                 "maxToken": 8192
+        #             },
+        #             "toolConfig": "none",
+        #             "concurrency": "",
+        #             "outputType": "STRING",
+        #             "isPromptSelected": True,
+        #             "isSmartPromptSelected": False,
+        #             "dependency": dependencies,  # Use the dependencies extracted from promptText
+        #             "columnID": columnID,  # Use the generated column ID
+        #             "label": col,
+        #             "type": "PROMPT",
+        #             "order": indx,
+        #         }
+
+        if col.startswith('context') and dataStreamName != None :
             if queryColName and dataStreamName:
                 dependencies = []
                 dependencies.append(columnIDMapping[queryColName])
@@ -312,9 +313,9 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
 
             dependencies.append(columnIDMapping[outputColName])  # Add the output column ID
 
-            longDef = definationMapping.get(col, {}).get('definition', "")
-            shortDef =definationMapping.get(col, {}).get('briefDefinition', "")
-            enum =  col.upper().replace(" ","_")
+            longDef = definationMapping.get(col.rsplit("_",1)[0], {}).get('definition', "")
+            shortDef =definationMapping.get(col.rsplit("_",1)[0], {}).get('briefDefinition', "")
+            enum = col.rsplit("_",1)[0].upper().replace(" ","_")
 
             template = {
       "analytics": [
@@ -322,20 +323,23 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
       ],
       "evaluationMetric": "ALL",
       "evaluationModel": "LLUMO_EVALLM",
-      "selectPrompt": columnIDMapping[outputColName],
+      "selectPrompt": None if "output" not in columnIDMapping.keys() else columnIDMapping["output"],
       "scoreCondition": "GREATER_THAN",
       "scoreValue": "50",
       "scoreResult": "PASS",
-      "llmKpi": col,
+      "llmKpi": col.rsplit("_",1)[0],
       "setRules": True,
       "type": "EVAL",
       "evalType": "LLM",
       "similarityMetric": None,
       "embeddingModel": None,
-      "groundTruth": None,
+      "groundTruth": None if "groundTruth" not in columnIDMapping.keys() else columnIDMapping["groundTruth"],
       "dataStream": None,
-      "context": None,
-      "dependency": [columnIDMapping[outputColName]],
+      "context":None if "context" not in columnIDMapping.keys() else columnIDMapping["context"],
+      "dependency":[ columnIDMapping[dep] for dep in  evalDependencies[ col.rsplit("_",1)[0]]],
+      "query": None if "query" not in columnIDMapping.keys() else columnIDMapping["query"],
+    "tools":None if "tools" not in columnIDMapping.keys() else columnIDMapping["tools"],
+    "messageHistory":None if "messageHistory" not in columnIDMapping.keys() else columnIDMapping["messageHistory"],
       "hallucinationFields": {
         "query": None,
         "context": None,
@@ -344,7 +348,7 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
       "definition": longDef,
       "analyticsENUM": enum,
       "prompt": shortDef,
-      "analyticsName": col,
+      "analyticsName": col.rsplit("_",1)[0],
       "columnID": columnID,
       "label": col,
       "order": indx
@@ -452,9 +456,12 @@ def uploadRowsInDBPlayground(payload):
         return None
 
 
-def createPlayground(email, workspaceID, df, promptText=None,queryColName=None,dataStreamName=None,definationMapping=None,outputColName="output",evalOutputMap = None):
+def createPlayground(email, workspaceID, df, promptText=None,queryColName=None,dataStreamName=None,definationMapping=None,outputColName="output",evalOutputMap = None,activePlayground=None):
     
-    playgroundId = str(createEvalPlayground(email=email, workspaceID=workspaceID))
+    if activePlayground != None:
+        playgroundId=activePlayground
+    else:
+        playgroundId = str(createEvalPlayground(email=email, workspaceID=workspaceID))
     payload1, payload2 = createColumn(
         workspaceID=workspaceID, dataframe=df, playgroundID=playgroundId, promptText=promptText,queryColName=queryColName,dataStreamName=dataStreamName,definationMapping=definationMapping,outputColName=outputColName,evalOutputMap=evalOutputMap
     )
@@ -472,7 +479,7 @@ def createPlayground(email, workspaceID, df, promptText=None,queryColName=None,d
 
 
 
-def getPlaygroundInsights(workspaceID: str, activePlayground: str):
+def getPlaygroundInsights(defination:str,uniqueClassesString: str, reasonList: list):
     headers = {
         
         "Content-Type": "application/json",
@@ -480,48 +487,29 @@ def getPlaygroundInsights(workspaceID: str, activePlayground: str):
 
     # Initial request to generate playground insights
     payload = {
-        "activePlayground": activePlayground,
-        "workspaceID": workspaceID,
+        "uniqueClassesString": uniqueClassesString,
+        "reasonList": reasonList,
+        "definition": defination,
     }
 
     urlGenerate = createInsightUrl
-    
-    responseGenerate = requests.post(urlGenerate, json=payload, headers=headers)
-
-    if responseGenerate.status_code == 200:
-        responseJson = responseGenerate.json()
-        
-        insightStatus = responseJson.get("data", {}).get("insight", False)
+    try:
+        responseGenerate = requests.post(urlGenerate, json=payload, headers=headers)
 
-        if insightStatus:
-            # If insight is true, request to get all playground insights
-            urlGetAll = getPlaygroundInsightsUrl
+        if responseGenerate.status_code == 200:
+            responseJson = responseGenerate.json()
             
-            responseGetAll = requests.post(urlGetAll, json=payload, headers=headers)
-
-            if responseGetAll.status_code == 200:
-                responseAllJson = responseGetAll.json()
-                
-                data = responseAllJson.get("data", {}).get("data", [])
-                
-                # Extract insight and solution
-                insights = []
-                for item in data:
-                    insight = item.get("insight", "")
-                    solution = item.get("solution", "")
-                    insights.append({"insight": insight, "solution": solution})
-
-                return insights
-            else:
-                print(f"Error fetching all insights: {responseGetAll.status_code} - {responseGetAll.text}")
-                return None
-        else:
-            print("No insight generated.")
-            return None
+            filteredResponse = {key: value for key, value in responseJson.items() if key in ['analysis', 'nextStep']}
+            
+            return filteredResponse
+    except Exception as e:
+        print(f"Exception occurred while generating insight: {e}")
+        return None
+        
     else:
         print(f"Error generating insight: {responseGenerate.status_code} - {responseGenerate.text}")
         return None
-def checkDependency(selectedEval, columns,tocheck=True):
+def checkDependency(selectedEval:list = [], columns:list = [],tocheck=True,_returnDepMapping = False):
     """
     Checks if all the required input columns for the selected evaluation metric are present.
 
@@ -532,33 +520,35 @@ def checkDependency(selectedEval, columns,tocheck=True):
     Raises:
     - LlumoAIError.dependencyError: If any required column is missing.
     """
-    if tocheck:
     # Define required dependencies for each evaluation metric
-        metricDependencies = {
-            'Response Completeness': ['context', 'query', 'output'],
-            'Response Bias': ['output'],
-            'Response Harmfulness': ['output'],
-            'Input Toxicity': ['query'],
-            'Input Harmfulness': ['query'],
-            'Context Utilization': ['output', 'context'],
-            'Relevance Retention': ['context', 'query'],
-            'Semantic Cohesion': ['context'],
-            'Final Task Alignment': ['messageHistory'],
-            'Tool Reliability': ['messageHistory'],
-            'Response Correctness': ['output', 'query', 'context'],
-            'Response Toxicity': ['output'],
-            'Input Bias': ['query'],
-            'Input Relevancy': ['context', 'query'],
-            'Redundancy Reduction': ['context'],
-            'Response Sentiment': ['output'],
-            'Tool Selection Accuracy': ['tools', 'messageHistory'],
-            'Stepwise Progression': ['tools', 'messageHistory'],
-            'Hallucination': ['query', 'context', 'output'],
-            'Groundedness': ['groundTruth', 'output'],
-            'Memory Utilization': ['context', 'messageHistory'],
-            'Input Relevancy (Multi-turn)': ['context', 'query']
-        }
+    metricDependencies = {
+        'Response Completeness': ['context', 'query', 'output'],
+        'Response Bias': ['output'],
+        'Response Harmfulness': ['output'],
+        'Input Toxicity': ['query'],
+        'Input Harmfulness': ['query'],
+        'Context Utilization': ['output', 'context'],
+        'Relevance Retention': ['context', 'query'],
+        'Semantic Cohesion': ['context'],
+        'Final Task Alignment': ['messageHistory'],
+        'Tool Reliability': ['messageHistory'],
+        'Response Correctness': ['output', 'query', 'context'],
+        'Response Toxicity': ['output'],
+        'Input Bias': ['query'],
+        'Input Relevancy': ['context', 'query'],
+        'Redundancy Reduction': ['context'],
+        'Response Sentiment': ['output'],
+        'Tool Selection Accuracy': ['tools', 'messageHistory'],
+        'Stepwise Progression': ['tools', 'messageHistory'],
+        'Hallucination': ['query', 'context', 'output'],
+        'Groundedness': ['groundTruth', 'output'],
+        'Memory Utilization': ['context', 'messageHistory'],
+        'Input Relevancy (Multi-turn)': ['context', 'query']
+    }
+    if _returnDepMapping == True:
+        return metricDependencies
 
+    if tocheck == True:
         # Check if the selected evaluation metric is known
         if selectedEval not in metricDependencies:
             return {"status": False,"message":f"Unknown evaluation metric: {selectedEval}"}
@@ -646,8 +636,44 @@ def validateOpenaiKey(api_key):
 def validateGoogleKey(api_key):
     try:
         genai.configure(api_key=api_key)
-        _ = genai.GenerativeModel("gemini-2.0").generate_content("test")
+        _ = genai.GenerativeModel("gemini-2.0-flash-lite").generate_content("test")
     except Exception as e:
         if "PERMISSION_DENIED" in str(e) or "API key not valid" in str(e):
             raise ValueError("❌ Invalid Google API key.")
         raise RuntimeError(f"⚠️ Error validating Gemini key: {e}")
+
+def groupLogsByClass(logs, max_logs=2):
+    # Initialize the final result structures (no defaultdict)
+    groupedLogs = {}
+    uniqueEdgeCases = {}  # This will store unique edge cases for each eval_name
+
+    # Iterate through the logs
+    for log in logs:
+        log_details = list(log.values())[0]  # Get the details dictionary
+        eval_name = log_details.get("kpi", "unmarked")
+        edge_case = log_details.get("edgeCase", "unmarked")
+        reasoning = log_details.get("reasoning", "")
+
+        if eval_name != "unmarked" and edge_case != "unmarked":
+            # Ensure that the eval_name and edge_case exist in the dictionary
+            if eval_name not in groupedLogs:
+                groupedLogs[eval_name] = {}
+                uniqueEdgeCases[eval_name] = set()  # Initialize the set for unique edge cases
+
+            if edge_case not in groupedLogs[eval_name]:
+                groupedLogs[eval_name][edge_case] = []
+
+            # Append the reasoning to the correct place
+            groupedLogs[eval_name][edge_case].append(reasoning)
+            uniqueEdgeCases[eval_name].add(edge_case)  # Add the edge case to the set
+
+    # Limit the number of reasons to max_logs
+    for eval_name in groupedLogs:
+        for edge_case in groupedLogs[eval_name]:
+            groupedLogs[eval_name][edge_case] = groupedLogs[eval_name][edge_case][:max_logs]
+
+    # Convert the set of unique edge cases to a list for easier reading
+    for eval_name in uniqueEdgeCases:
+        uniqueEdgeCases[eval_name] = list(uniqueEdgeCases[eval_name])
+
+    return groupedLogs, uniqueEdgeCases

llumo/openai.py

@@ -0,0 +1,79 @@
+from openai import OpenAI as OpenAIClient
+from .client import LlumoClient
+
+# Dummy evaluation function that uses LlumoClient
+def evaluate_multiple(data, api_key=None,evals=["Response Correctness"]):
+    client = LlumoClient(api_key=api_key)
+    results= client.evaluateMultiple(data, evals=evals,createExperiment=False,prompt_template="Give answer to the query: {{query}}, using context: {{context}}",getDataFrame=False)
+    print(results)
+    return results
+
+# Wrapper around ChatCompletion to allow custom fields like `.evaluation`
+class ChatCompletionWithEval:
+    def __init__(self, response, evaluation):
+        self._response = response
+        self.evaluation = evaluation
+
+    def __getattr__(self, name):
+        return getattr(self._response, name)
+
+    def __getitem__(self, key):
+        return self._response[key]
+
+    def __repr__(self):
+        return repr(self._response)
+
+class openai(OpenAIClient):
+    def __init__(self, api_key: str):
+        super().__init__(api_key=api_key)
+
+        original_create = self.chat.completions.create
+
+        class ChatCompletionsWrapper:
+            @staticmethod
+            def create(*args, **kwargs):
+                context = kwargs.pop("context", None)
+                evals = kwargs.pop("evals", [])
+                llumo_key = kwargs.pop("llumo_key", None)
+
+                messages = kwargs.get("messages", [])
+                user_message = next(
+                    (m.get("content") for m in reversed(messages) if m.get("role") == "user"),
+                    "",
+                )
+
+                # If context is None or empty or whitespace-only, set it to user_message
+                if not context or context.strip() == "":
+                    context = user_message
+
+                response = original_create(*args, **kwargs)
+
+                try:
+                    output_text = response.choices[0].message.content
+                except Exception:
+                    output_text = ""
+
+                eval_input = [{
+                    "query": user_message,
+                    "context": context,
+                    "output": output_text,
+                }]
+
+                # Safely call evaluate_multiple, if error return None
+                evaluation = None
+                try:
+                    evaluation = evaluate_multiple(eval_input, api_key=llumo_key,evals=evals)
+                except Exception as e:
+                    # You can optionally log the error here if you want
+                    # print(f"Evaluation failed, skipping: {e}")
+                    evaluation = None
+
+                # If evaluation is None, just return normal response
+                if evaluation is None:
+                    print("All server are busy for evaluation ")
+                    return response
+
+                # Otherwise wrap with evaluation attached
+                return ChatCompletionWithEval(response, evaluation)
+
+        self.chat.completions.create = ChatCompletionsWrapper.create

{llumo-0.2.15b2.dist-info → llumo-0.2.16b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.15b2
+Version: 0.2.16b1
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

llumo-0.2.16b1.dist-info/RECORD

@@ -0,0 +1,16 @@
+llumo/__init__.py,sha256=YVBkF1fiXFBd_zzySi9BDWgX8MJuLBJ-oF8538MrnDU,256
+llumo/chains.py,sha256=6lCgLseh04RUgc6SahhmvQj82quay2Mi1j8gPUlx8Es,2923
+llumo/client.py,sha256=rOTbw8QGi5CnQ77QKS4rKh-dSBSVoyVAORrK1i_b5EQ,60339
+llumo/exceptions.py,sha256=Vp_MnanHbnd1Yjuoi6WLrKiwwZbJL3znCox2URMmGU4,2032
+llumo/execution.py,sha256=nWbJ7AvWuUPcOb6i-JzKRna_PvF-ewZTiK8skS-5n3w,1380
+llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
+llumo/google.py,sha256=5AVAqxPN20UuHIqi4yuHHSTf49LI96krtbztJ5qt8L0,1413
+llumo/helpingFuntions.py,sha256=0W2JNdLyOV92lgESgB_JyJmOUvW5ooRdZyjN5LKDSX0,25296
+llumo/models.py,sha256=aVEZsOOoQx5LeNtwSyBxqvrINq0izH3QWu_YjsMPE6o,2910
+llumo/openai.py,sha256=BEmsOdHiQzDpKv6b4L62JaUMq7DbpICNPqyfMNRWi2I,2981
+llumo/sockets.py,sha256=I2JO_eNEctRo_ikgvFVp5zDd-m0VDu04IEUhhsa1Tic,5950
+llumo-0.2.16b1.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
+llumo-0.2.16b1.dist-info/METADATA,sha256=_e94VIPrn02CP0X9gdkICA210Te_inzaSPcfH0p-Hlk,1521
+llumo-0.2.16b1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+llumo-0.2.16b1.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
+llumo-0.2.16b1.dist-info/RECORD,,

llumo-0.2.15b2.dist-info/RECORD

@@ -1,13 +0,0 @@
-llumo/__init__.py,sha256=O04b4yW1BnOvcHzxWFddAKhtdBEhBNhLdb6xgnpHH_Q,205
-llumo/client.py,sha256=60RSxhk-9wzK9KgBz8dfbUd3-AaKiljxqbHI5UL8GIw,54021
-llumo/exceptions.py,sha256=Vp_MnanHbnd1Yjuoi6WLrKiwwZbJL3znCox2URMmGU4,2032
-llumo/execution.py,sha256=nWbJ7AvWuUPcOb6i-JzKRna_PvF-ewZTiK8skS-5n3w,1380
-llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
-llumo/helpingFuntions.py,sha256=BZfUIgTO0PJchppHn0wDRF1wcYSuMST5ry95HBPN5SQ,23534
-llumo/models.py,sha256=aVEZsOOoQx5LeNtwSyBxqvrINq0izH3QWu_YjsMPE6o,2910
-llumo/sockets.py,sha256=I2JO_eNEctRo_ikgvFVp5zDd-m0VDu04IEUhhsa1Tic,5950
-llumo-0.2.15b2.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
-llumo-0.2.15b2.dist-info/METADATA,sha256=vbXwSwhuxnO0CSMz4uJ45AepuwVMl7irZlHmYkqRYbY,1521
-llumo-0.2.15b2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-llumo-0.2.15b2.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
-llumo-0.2.15b2.dist-info/RECORD,,