llumo 0.2.16b2__py3-none-any.whl → 0.2.17__py3-none-any.whl

llumo/client.py

@@ -406,6 +406,11 @@ class LlumoClient:
 
     def compressor(self, data, prompt_template):
         results = []
+        if isinstance(data, dict):
+            data = [data]
+        elif not isinstance(data, list):
+            raise ValueError("Data should be a dict or a list of dicts.")
+
         dataframe = pd.DataFrame(data)
 
         try:
@@ -612,12 +617,17 @@ class LlumoClient:
         self,
         data,
         evals: list,  # list of eval metric names
-        prompt_template="",
+        prompt_template="Give answer to the given query: {{query}} using the given context: {{context}}.",
         outputColName="output",
         createExperiment: bool = False,
         getDataFrame:bool =False,
         _tocheck=True,
     ):
+        if isinstance(data, dict):
+            data = [data]
+        elif not isinstance(data, list):
+            raise ValueError("Data should be a dict or a list of dicts.")
+
         self.socket = LlumoSocketClient(socketUrl)
         dataframe = pd.DataFrame(data).astype(str)
         workspaceID = None
@@ -868,130 +878,76 @@ class LlumoClient:
     def promptSweep(
         self,
         templates: List[str],
-        dataset: Dict[str, List[str]],
+        data,
         model_aliases: List[AVAILABLEMODELS],
         apiKey: str,
         evals=["Response Correctness"],
         toEvaluate: bool = False,
         createExperiment: bool = False,
-        getDataFrame = False
-
-
+        getDataFrame=False
     ) -> pd.DataFrame:
-
+        if isinstance(data, dict):
+                data = [data]
+            # Check if data is now a list of dictionaries
+        if isinstance(data, list) and all(isinstance(item, dict) for item in data):
+            working_df= pd.DataFrame(data).astype(str)
+        else:
+            raise ValueError("Data must be a dictionary or a list of dictionaries.")
         modelStatus = validateModels(model_aliases=model_aliases)
-        if modelStatus["status"]== False:
+        if modelStatus["status"] == False:
             raise LlumoAIError.providerError(modelStatus["message"])
 
         self.validateApiKey()
         workspaceID = self.workspaceID
         email = self.email
         executor = ModelExecutor(apiKey)
-        keys = list(dataset.keys())
-        value_combinations = list(itertools.product(*dataset.values()))
-        combinations = [dict(zip(keys, values)) for values in value_combinations]
-
-        results = []
-
-        for combo in combinations:
-            for template in templates:
-                prompt = template
-                for k, v in combo.items():
-                    prompt = prompt.replace(f"{{{{{k}}}}}", v)
-
-                row = {
-                    "prompt": prompt,
-                    **combo,
-                }
-
-                for i, model in enumerate(model_aliases, 1):
-                    try:
-                        provider = getProviderFromModel(model)
-                        response = executor.execute(
-                            provider, model.value, prompt, apiKey
-                        )
-                        outputKey = f"output_{i}"
-                        row[outputKey] = response
-                    except Exception as e:
-                        row[f"output_{i}"] = str(e)
-
-                results.append(row)
-
-        df = pd.DataFrame(results)
-
-        if toEvaluate == True:
-            dfWithEvals = df.copy()
-            for i, model in enumerate(model_aliases, 1):
-                outputColName = f"output_{i}"
-                try:
-                    res = self.evaluateMultiple(
-                        df.to_dict("records"),
-                        evals=evals,
-                        prompt_template=str(templates[0]),
-                        outputColName=outputColName,
-                        _tocheck=False,
-                        getDataFrame=True,
-                        createExperiment = False
-                    )
-
-                    # Rename all new columns with _i+1 (e.g., _1, _2)
-                    for evalMetric in evals:
-                        scoreCol = f"{evalMetric}"
-                        reasonCol = f"{evalMetric} Reason"
-                        if scoreCol in res.columns:
-                            res = res.rename(columns={scoreCol: f"{scoreCol}_{i}"})
-                        if reasonCol in res.columns:
-                            res = res.rename(columns={reasonCol: f"{reasonCol}_{i}"})
-
-                    # Drop duplicated columns from df (like prompt, variables, etc.)
-                    newCols = [
-                        col for col in res.columns if col not in dfWithEvals.columns
-                    ]
-                    dfWithEvals = pd.concat([dfWithEvals, res[newCols]], axis=1)
+        prompt_template = templates[0]
+        
+        working_df = self._outputForStream(working_df, model_aliases, prompt_template, apiKey)
 
-                except Exception as e:
-                    print(f"Evaluation failed for model {model.value}: {str(e)}")
+        # Optional evaluation
+        outputEvalMapping = None
+        if toEvaluate:
+            for evalName in evals:
+                # Validate API and dependencies
+                self.validateApiKey(evalName=evalName)
+                metricDependencies = checkDependency(
+                    evalName, list(working_df.columns), tocheck=False
+                )
+                if not metricDependencies["status"]:
+                    raise LlumoAIError.dependencyError(metricDependencies["message"])
 
-            if createExperiment:
-                pd.set_option("future.no_silent_downcasting", True)
-                dfWithEvals = dfWithEvals.fillna("Some error occurred")
-                if createPlayground(
-                    email,
-                    workspaceID,
-                    dfWithEvals,
-                    promptText=templates[0],
+            working_df, outputEvalMapping = self._evaluateForStream(working_df, evals, model_aliases, prompt_template,generateOutput=True)
+        if createExperiment:
+            # df = working_df.fillna("Some error occured").astype(object)
+            with warnings.catch_warnings():
+                warnings.simplefilter(action='ignore', category=FutureWarning)
+                df = working_df.fillna("Some error occurred").astype(str)
+            if createPlayground(
+                    email, workspaceID, df,
+                    promptText=prompt_template,
                     definationMapping=self.definationMapping,
-                ):
+                    evalOutputMap=outputEvalMapping
+            ):
+                print(
+                    "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results.")
+        else:
+            if getDataFrame == True and toEvaluate == True:
+                return LlumoDataFrameResults(working_df, evals=self.evals, evalData=self.evalData,
+                                             definationMapping=self.definationMapping)
 
-                    print(
-                        "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results."
-                    )
-            else:
-                if getDataFrame:
-                    return LlumoDataFrameResults(dfWithEvals, evals=self.evals, evalData=self.evalData,
-                                                 definationMapping=self.definationMapping)
-                else:
-                    data = dfWithEvals.to_dict(orient="records")
-                    return LlumoDictResults(data, evals=self.evals, evalData=self.evalData,definationMapping=self.definationMapping)
+            elif getDataFrame == False and toEvaluate == True:
+                data = working_df.to_dict(orient="records")
+                return LlumoDictResults(data, evals=self.evals, evalData=self.evalData,
+                                        definationMapping=self.definationMapping)
 
+            elif getDataFrame== True and toEvaluate == False:
+                return working_df
+
+            elif getDataFrame == False and toEvaluate == False :
+                return working_df.to_dict(orient = "records")
 
-        else:
-            if createExperiment == True:
-                pd.set_option("future.no_silent_downcasting", True)
-                df = df.fillna("Some error occurred")
 
-                if createPlayground(email, workspaceID, df, promptText=templates[0]):
-                    print(
-                        "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results."
-                    )
-            else:
-                if getDataFrame:
-                    return LlumoDataFrameResults(df, evals=self.evals, evalData=self.evalData,
-                                                 definationMapping=self.definationMapping)
-                else:
-                    data = df.to_dict(orient="records")
-                    return LlumoDictResults(data, evals=self.evals, evalData=self.evalData,
-                                            definationMapping=self.definationMapping)
 
 
 
@@ -1008,6 +964,11 @@ class LlumoClient:
         getDataFrame:bool = False
 
     ):
+        if isinstance(data, dict):
+            data = [data]
+        elif not isinstance(data, list):
+            raise ValueError("Data should be a dict or a list of dicts.")
+
         if model.lower() not in ["openai", "google"]:
             raise ValueError("Model must be 'openai' or 'google'")
 
@@ -1057,6 +1018,10 @@ class LlumoClient:
         outputColName="output"
 
     ):
+        if isinstance(data, dict):
+            data = [data]
+        elif not isinstance(data, list):
+            raise ValueError("Data should be a dict or a list of dicts.")
 
         dataframe = pd.DataFrame(data)
 
@@ -1103,6 +1068,11 @@ class LlumoClient:
             generateOutput=True,
             getDataFrame = False
     ):
+        if isinstance(data, dict):
+            data = [data]
+        elif not isinstance(data, list):
+            raise ValueError("Data should be a dict or a list of dicts.")
+
         # Validate required parameters
         if generateOutput:
             if not modelAliases:
@@ -1238,7 +1208,6 @@ class LlumoClient:
 
         
         self.socket.disconnect()
-
         # Create experiment if required
         if createExperiment:
             # df = working_df.fillna("Some error occured").astype(object)
@@ -1368,6 +1337,10 @@ class LlumoClient:
             getDataFrame = False
            ):
 
+        if isinstance(data, dict):
+            data = [data]
+        elif not isinstance(data, list):
+            raise ValueError("Data should be a dict or a list of dicts.")
 
         # Copy the original dataframe
         original_df = pd.DataFrame(data)
@@ -1508,7 +1481,7 @@ class LlumoClient:
         except Exception as e:
             raise "Some error ocuured please check your API key"
 
-    def upload(self, file_path):
+    def uploadfile(self, file_path):
 
         workspaceID = None
         email = None
@@ -1550,8 +1523,97 @@ class LlumoClient:
 
         except Exception as e:
             print(f"Error: {e}")
+            
+    def upload(self,data):
+        try:
+            if isinstance(data, dict):
+                data = [data]
+            # Check if data is now a list of dictionaries
+            if isinstance(data, list) and all(isinstance(item, dict) for item in data):
+                dataframe = pd.DataFrame(data).astype(str)
+            else:
+                raise ValueError("Data must be a dictionary or a list of dictionaries.")
+            self.validateApiKey()
+            if createPlayground(self.email, self.workspaceID, dataframe):
+                print(
+                    "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results."
+                )
+                return True
+
+        except Exception as e:
+            print(f"Error: {e}")
+            return False
+
+
+    def createExperimentWithEvals(
+        self,
+        data,
+        evals: list,  # list of eval metric names
+        prompt_template="Give answer to the given query: {{query}} using the given context: {{context}}.",
+        outputColName="output",
+        createExperiment: bool = False,
+        getDataFrame:bool =False,
+        _tocheck=True,
+    ):
+        if isinstance(data, dict):
+            data = [data]
+        elif not isinstance(data, list):
+            raise ValueError("Data should be a dict or a list of dicts.")
+        print("reciving data")
+        dataframe = pd.DataFrame(data).astype(str)
+        workspaceID = None
+        email = None
+        self.evalData=[]
+        self.evals=evals
+        self.allBatches = []
+        rowIdMapping = {}  # (rowID-columnID-columnID -> (index, evalName))
+        print("single eval validate")
+        self.validateApiKey(evalName=evals[0])
+        if createExperiment:
+            activePlayground = str(createEvalPlayground(email=self.email, workspaceID=self.workspaceID))
+            
+        else:    
+            activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
+                        "-", ""
+                    )        
+        for evalName in evals:
+            print("validate in loop")
+            self.validateApiKey(evalName=evalName)
 
+    
+        print("convertinf to dict")
+        self.evalData =dataframe.to_dict(orient="records") 
+
+        if createExperiment:
+            print("heading to upload")
+            pd.set_option("future.no_silent_downcasting", True)
+            # df = dataframe.fillna("Some error occured").astype(object)
+            with warnings.catch_warnings():
+                warnings.simplefilter(action='ignore', category=FutureWarning)
+                df = dataframe.fillna("Some error occurred").astype(str)
+
+            df = dataframe.fillna("Some error occured").infer_objects(copy=False)
+            if createPlayground(
+                self.email,
+                self.workspaceID,
+                df,
+                promptText=prompt_template,
+                definationMapping=self.definationMapping,
+                outputColName=outputColName,
+                activePlayground= activePlayground
+            ):
+                print(
+                    "LLUMO’s intuitive UI is ready—start exploring and experimenting with your logs now. Visit https://llumo.ai/evallm to see the results."
+                )
+                
+        else:
+            if getDataFrame:
+                return LlumoDataFrameResults(dataframe,evals=self.evals,evalData=self.evalData,definationMapping=self.definationMapping)
+            else:
+                data=dataframe.to_dict(orient="records")
+                return LlumoDictResults(data,evals=self.evals,evalData=self.evalData,definationMapping=self.definationMapping)
 
+    
 class SafeDict(dict):
     def __missing__(self, key):
         return ""

llumo/google.py

@@ -59,7 +59,7 @@ class genai:
                 evaluation = None
             
             if evaluation is None:
-                    print("Cannot process your request, please check your api and try again later.")
+                    print("Cannot process your request for evaluation, please check your api and try again later.")
                     return response
             
 

llumo/helpingFuntions.py

@@ -233,8 +233,7 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
 
     # Create a mapping of column names to unique column IDs
     columnIDMapping = {}
-    print("Def Mapping: ")
-    print(definationMapping)
+
     # Iterate over each column in the dataframe
     for indx, col in enumerate(dataframe.columns):
         # Generate a unique column ID using uuid
@@ -336,7 +335,7 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
       "groundTruth": None if "groundTruth" not in columnIDMapping.keys() else columnIDMapping["groundTruth"],
       "dataStream": None,
       "context":None if "context" not in columnIDMapping.keys() else columnIDMapping["context"],
-      "dependency":[ columnIDMapping[dep] for dep in  evalDependencies[ col.rsplit("_",1)[0]]],
+      "dependency":[columnIDMapping[col] if dep == "output" else columnIDMapping[dep] for dep in evalDependencies[col.rsplit("_", 1)[0]]],
       "query": None if "query" not in columnIDMapping.keys() else columnIDMapping["query"],
     "tools":None if "tools" not in columnIDMapping.keys() else columnIDMapping["tools"],
     "messageHistory":None if "messageHistory" not in columnIDMapping.keys() else columnIDMapping["messageHistory"],

{llumo-0.2.16b2.dist-info → llumo-0.2.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.16b2
+Version: 0.2.17
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

llumo-0.2.17.dist-info/RECORD

@@ -0,0 +1,16 @@
+llumo/__init__.py,sha256=YVBkF1fiXFBd_zzySi9BDWgX8MJuLBJ-oF8538MrnDU,256
+llumo/chains.py,sha256=6lCgLseh04RUgc6SahhmvQj82quay2Mi1j8gPUlx8Es,2923
+llumo/client.py,sha256=LhvBPxKCbt1TXcrmDbd7u_hOXldiC34NHTlRwlAG5VI,63133
+llumo/exceptions.py,sha256=Vp_MnanHbnd1Yjuoi6WLrKiwwZbJL3znCox2URMmGU4,2032
+llumo/execution.py,sha256=nWbJ7AvWuUPcOb6i-JzKRna_PvF-ewZTiK8skS-5n3w,1380
+llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
+llumo/google.py,sha256=3S_aRtbtlctCXPGR0u4baLlkyFrsjd02vlUCkoRPA5U,2147
+llumo/helpingFuntions.py,sha256=ORBM5xPPMHc3ENnJOg4i3Enxtor4En5oDj1d0Zu1mLk,25284
+llumo/models.py,sha256=aVEZsOOoQx5LeNtwSyBxqvrINq0izH3QWu_YjsMPE6o,2910
+llumo/openai.py,sha256=DGhEwQIJIIycGpw3hYQnyxdj6RFVpZ-gay-fZGqtkhU,3013
+llumo/sockets.py,sha256=I2JO_eNEctRo_ikgvFVp5zDd-m0VDu04IEUhhsa1Tic,5950
+llumo-0.2.17.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
+llumo-0.2.17.dist-info/METADATA,sha256=3rw1KKwWOUFZMT3JaBAqGAugsPxB011FhGGccVBCNTY,1519
+llumo-0.2.17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+llumo-0.2.17.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
+llumo-0.2.17.dist-info/RECORD,,

llumo-0.2.16b2.dist-info/RECORD

@@ -1,16 +0,0 @@
-llumo/__init__.py,sha256=YVBkF1fiXFBd_zzySi9BDWgX8MJuLBJ-oF8538MrnDU,256
-llumo/chains.py,sha256=6lCgLseh04RUgc6SahhmvQj82quay2Mi1j8gPUlx8Es,2923
-llumo/client.py,sha256=rOTbw8QGi5CnQ77QKS4rKh-dSBSVoyVAORrK1i_b5EQ,60339
-llumo/exceptions.py,sha256=Vp_MnanHbnd1Yjuoi6WLrKiwwZbJL3znCox2URMmGU4,2032
-llumo/execution.py,sha256=nWbJ7AvWuUPcOb6i-JzKRna_PvF-ewZTiK8skS-5n3w,1380
-llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
-llumo/google.py,sha256=F-n7DCQ7mvORGyK_yGliL14N0AGyVGmjKgO60FqOOKI,2132
-llumo/helpingFuntions.py,sha256=0W2JNdLyOV92lgESgB_JyJmOUvW5ooRdZyjN5LKDSX0,25296
-llumo/models.py,sha256=aVEZsOOoQx5LeNtwSyBxqvrINq0izH3QWu_YjsMPE6o,2910
-llumo/openai.py,sha256=DGhEwQIJIIycGpw3hYQnyxdj6RFVpZ-gay-fZGqtkhU,3013
-llumo/sockets.py,sha256=I2JO_eNEctRo_ikgvFVp5zDd-m0VDu04IEUhhsa1Tic,5950
-llumo-0.2.16b2.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
-llumo-0.2.16b2.dist-info/METADATA,sha256=5Nys3GS9XTD0fE9K4_1IxjD2YG1JpC1Fl2M1q93orf4,1521
-llumo-0.2.16b2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-llumo-0.2.16b2.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
-llumo-0.2.16b2.dist-info/RECORD,,