llumo 0.2.14b6__py3-none-any.whl → 0.2.15b1__py3-none-any.whl

llumo/client.py

@@ -25,6 +25,11 @@ postUrl = (
 fetchUrl = (
     "https://red-skull-service-392377961931.us-central1.run.app/api/get-cells-data"
 )
+socketDataUrl = "https://app.llumo.ai/api/eval/get-awaited"
+# {
+#     "workspaceID":"c9191fdf33bdd7838328c1a0",
+#     "playgroundID":"17496117244856b7815ac94004347b1c2e2f7e01600ec"
+# }
 validateUrl = "https://app.llumo.ai/api/workspace-details"
 socketUrl = "https://red-skull-service-392377961931.us-central1.run.app/"
 
@@ -33,7 +38,7 @@ class LlumoClient:
 
     def __init__(self, api_key):
         self.apiKey = api_key
-        self.socket = LlumoSocketClient(socketUrl)
+        
         self.processMapping = {}
         self.definationMapping = {}
 
@@ -45,6 +50,7 @@ class LlumoClient:
         reqBody = {"analytics": [evalName]}
 
         try:
+            print(reqBody)
             response = requests.post(url=validateUrl, json=reqBody, headers=headers)
 
         except requests.exceptions.RequestException as e:
@@ -89,9 +95,8 @@ class LlumoClient:
             )
             self.email = data["data"]["data"].get("email", None)
 
-            self.definationMapping[evalName] = data["data"]["data"]["analyticsMapping"][
-                evalName
-            ]
+            self.definationMapping[evalName] = data.get("data", {}).get("data", {}).get("analyticsMapping", {}).get(evalName, None)
+
         except Exception as e:
             # print(f"Error extracting data from response: {str(e)}")
             raise LlumoAIError.UnexpectedError(detail=str(e))
@@ -577,7 +582,8 @@ class LlumoClient:
         createExperiment: bool = False,
         _tocheck=True,
     ):
-        dataframe = pd.DataFrame(data)
+        self.socket = LlumoSocketClient(socketUrl)
+        dataframe = pd.DataFrame(data).astype(str)
         workspaceID = None
         email = None
         socketID = self.socket.connect(timeout=250)
@@ -602,13 +608,16 @@ class LlumoClient:
             kwargs={
                 "min_wait": 40,
                 "max_wait": timeout,
-                "inactivity_timeout": 200,
+                "inactivity_timeout": 10,
                 "expected_results": expectedResults,
             },
             daemon=True,
         )
         listener_thread.start()
-
+        
+        activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
+                    "-", ""
+                )        
         for evalName in evals:
             # print(f"\n======= Running evaluation for: {evalName} =======")
 
@@ -656,9 +665,7 @@ class LlumoClient:
                 inputDict = {key: row[key] for key in keys if key in row}
                 output = row.get(outputColName, "")
 
-                activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
-                    "-", ""
-                )
+                
                 rowID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
                 columnID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
 
@@ -751,15 +758,27 @@ class LlumoClient:
         time.sleep(3)
         listener_thread.join()
 
-        raw_results = self.socket.getReceivedData()
-
+        
+        rawResults = self.socket.getReceivedData()
+        
+        # print("data from db #####################",dataFromDb)
         # Fix here: keep full keys, do not split keys
-        received_rowIDs = {key for item in raw_results for key in item.keys()}
-        expected_rowIDs = set(rowIdMapping.keys())
-        missing_rowIDs = expected_rowIDs - received_rowIDs
+        receivedRowIDs = {key for item in rawResults for key in item.keys()}
+        expectedRowIDs = set(rowIdMapping.keys())
+        missingRowIDs = expectedRowIDs - receivedRowIDs 
         # print("All expected keys:", expected_rowIDs)
         # print("All received keys:", received_rowIDs)
-        print("Missing keys:", len(missing_rowIDs))
+        # print("Missing keys:", len(missingRowIDs))
+        missingRowIDs=list(missingRowIDs)
+        
+        if len(missingRowIDs) > 0:
+            dataFromDb=fetchData(workspaceID,activePlayground,missingRowIDs)
+            rawResults.extend(dataFromDb)
+            
+            
+            
+            
+            
 
         # Initialize dataframe columns for each eval
         for eval in evals:
@@ -767,7 +786,7 @@ class LlumoClient:
             dataframe[f"{eval} Reason"] = None
 
         # Map results to dataframe rows
-        for item in raw_results:
+        for item in rawResults:
             for compound_key, value in item.items():
                 if compound_key in rowIdMapping:
                     index = rowIdMapping[compound_key]["index"]
@@ -776,6 +795,7 @@ class LlumoClient:
                     dataframe.at[index, f"{evalName} Reason"] = value.get("reasoning")
 
         self.socket.disconnect()
+        
 
         if createExperiment:
             pd.set_option("future.no_silent_downcasting", True)
@@ -794,7 +814,7 @@ class LlumoClient:
         else:
             return dataframe
 
-    def run_sweep(
+    def promptSweep(
         self,
         templates: List[str],
         dataset: Dict[str, List[str]],
@@ -803,9 +823,15 @@ class LlumoClient:
         evals=["Response Correctness"],
         toEvaluate: bool = False,
         createExperiment: bool = False,
+
+
     ) -> pd.DataFrame:
 
-        self.validateApiKey(evalName="")
+        modelStatus = validateModels(model_aliases=model_aliases)
+        if modelStatus["status"]== False:
+            raise LlumoAIError.providerError(modelStatus["message"])
+
+        self.validateApiKey()
         workspaceID = self.workspaceID
         email = self.email
         executor = ModelExecutor(apiKey)
@@ -910,6 +936,7 @@ class LlumoClient:
         evals=["Final Task Alignment"],
         prompt_template="Give answer for the given query: {{query}}",
         createExperiment: bool = False,
+
     ):
         if model.lower() not in ["openai", "google"]:
             raise ValueError("Model must be 'openai' or 'google'")
@@ -940,7 +967,7 @@ class LlumoClient:
         if createExperiment:
             pd.set_option("future.no_silent_downcasting", True)
             df = toolResponseDf.fillna("Some error occured")
-            if createPlayground(self.email, self.workspaceID, df):
+            if createPlayground(self.email, self.workspaceID, df,promptText=prompt_template,definationMapping=self.definationMapping):
                 print(
                     "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
                 )
@@ -971,183 +998,367 @@ class LlumoClient:
             #     )
             toolResponseDf = self.evaluateMultiple(
                 toolResponseDf.to_dict(orient="records"),
-                eval=evals,
+                evals=evals,
                 prompt_template="Give answer for the given query: {{query}}",
                 outputColName=outputColName,
+                createExperiment=createExperiment
             )
-            return toolResponseDf
+            if createExperiment:
+                pass
+            else:
+                return toolResponseDf
 
         except Exception as e:
             raise e
 
-    def runDataStream(
-        self,
-        data,
-        streamName: str,
-        queryColName: str = "query",
-        createExperiment: bool = False,
+    def ragSweep(
+            self,
+            data,
+            streamName: str,
+            queryColName: str = "query",
+            createExperiment: bool = False,
+            modelAliases=[],
+            apiKey="",
+            prompt_template="Give answer to the given: {{query}} using the context:{{context}}",
+            evals=["Context Utilization"],
+            toEvaluate=False,
+            generateOutput=True
     ):
-        results = {}
-        dataframe = pd.DataFrame(data)
-        try:
-            socketID = self.socket.connect(timeout=150)
-            # Ensure full connection before proceeding
-            max_wait_secs = 20
-            waited_secs = 0
-            while not self.socket._connection_established.is_set():
-                time.sleep(0.1)
-                waited_secs += 0.1
-                if waited_secs >= max_wait_secs:
-                    raise RuntimeError(
-                        "Timeout waiting for server 'connection-established' event."
-                    )
-            # print(f"Connected with socket ID: {socketID}")
-            rowIdMapping = {}
+        # Validate required parameters
+        if generateOutput:
+            if not modelAliases:
+                raise ValueError("Model aliases must be provided when generateOutput is True.")
+            if not apiKey or not isinstance(apiKey, str) or apiKey.strip() == "":
+                raise ValueError("Valid API key must be provided when generateOutput is True.")
+
+        modelStatus = validateModels(model_aliases=modelAliases)
+        if modelStatus["status"]== False:
+            if len(modelAliases) ==  0:
+                raise LlumoAIError.providerError("No model selected.")
+            else:
+                raise LlumoAIError.providerError(modelStatus["message"])
 
-            # print(f"Validating API key...")
-            self.validateApiKey()
-            # print(f"API key validation successful. Hits available: {self.hitsAvailable}")
+        # Copy the original dataframe
+        original_df = pd.DataFrame(data)
+        working_df = original_df.copy()
 
-            # check for available hits and trial limit
-            userHits = checkUserHits(
-                self.workspaceID,
-                self.hasSubscribed,
-                self.trialEndDate,
-                self.subscriptionEndDate,
-                self.hitsAvailable,
-                len(dataframe),
-            )
+        # Connect to socket
+        self.socket = LlumoSocketClient(socketUrl)
+        socketID = self.socket.connect(timeout=150)
+        waited_secs = 0
+        while not self.socket._connection_established.is_set():
+            time.sleep(0.1)
+            waited_secs += 0.1
+            if waited_secs >= 20:
+                raise RuntimeError("Timeout waiting for server 'connection-established' event.")
 
-            # do not proceed if subscription or trial limit has exhausted
-            if not userHits["success"]:
-                raise LlumoAIError.InsufficientCredits(userHits["message"])
+        self.validateApiKey()
 
-            print("====🚀Sit back while we fetch data from the stream 🚀====")
-            workspaceID = self.workspaceID
-            email = self.email
-            streamId = getStreamId(workspaceID, self.apiKey, streamName)
-            # Prepare all batches before sending
-            # print("Preparing batches...")
-            self.allBatches = []
-            currentBatch = []
+        # Check user credits
+        userHits = checkUserHits(
+            self.workspaceID, self.hasSubscribed, self.trialEndDate,
+            self.subscriptionEndDate, self.hitsAvailable, len(working_df)
+        )
+        if not userHits["success"]:
+            raise LlumoAIError.InsufficientCredits(userHits["message"])
 
-            for index, row in dataframe.iterrows():
-                activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
-                    "-", ""
-                )
-                rowID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
-                columnID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
+        print("====🚀Sit back while we fetch data from the stream 🚀====")
+        workspaceID, email = self.workspaceID, self.email
+        activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
+        streamId = getStreamId(workspaceID, self.apiKey, streamName)
 
-                rowIdMapping[rowID] = index
-                # Use the server-provided socket ID here
-                templateData = {
-                    "processID": getProcessID(),
-                    "socketID": socketID,
-                    "processData": {
-                        "executionDependency": {"query": row[queryColName]},
-                        "dataStreamID": streamId,
-                    },
-                    "workspaceID": workspaceID,
-                    "email": email,
-                    "type": "DATA_STREAM",
-                    "playgroundID": activePlayground,
-                    "processType": "DATA_STREAM",
-                    "rowID": rowID,
-                    "columnID": columnID,
-                    "source": "SDK",
-                }
+        # Prepare batches
+        rowIdMapping = {}
+        self.allBatches = []
+        currentBatch = []
 
-                currentBatch.append(templateData)
+        expectedResults = len(working_df)
+        timeout = max(100, min(150, expectedResults * 10))
+
+        listener_thread = threading.Thread(
+            target=self.socket.listenForResults,
+            kwargs={
+                "min_wait": 40,
+                "max_wait": timeout,
+                "inactivity_timeout": 10,
+                "expected_results": expectedResults,
+            },
+            daemon=True
+        )
+        listener_thread.start()
 
-            if len(currentBatch) == 10 or index == len(dataframe) - 1:
+        for index, row in working_df.iterrows():
+            rowID, columnID = uuid.uuid4().hex, uuid.uuid4().hex
+            compoundKey = f"{rowID}-{columnID}-{columnID}"
+            rowIdMapping[compoundKey] = {"index": index}
+            templateData = {
+                "processID": getProcessID(),
+                "socketID": socketID,
+                "processData": {
+                    "executionDependency": {"query": row[queryColName]},
+                    "dataStreamID": streamId,
+                },
+                "workspaceID": workspaceID,
+                "email": email,
+                "type": "DATA_STREAM",
+                "playgroundID": activePlayground,
+                "processType": "DATA_STREAM",
+                "rowID": rowID,
+                "columnID": columnID,
+                "source": "SDK",
+            }
+            currentBatch.append(templateData)
+            if len(currentBatch) == 10 or index == len(working_df) - 1:
                 self.allBatches.append(currentBatch)
                 currentBatch = []
 
-            # Post all batches
-            total_items = sum(len(batch) for batch in self.allBatches)
-            # print(f"Posting {len(self.allBatches)} batches ({total_items} items total)")
+        for batch in tqdm(self.allBatches, desc="Processing Batches", unit="batch", colour="magenta", ncols=80):
+            try:
+                self.postDataStream(batch=batch, workspaceID=workspaceID)
+                time.sleep(3)
+            except Exception as e:
+                print(f"Error posting batch: {e}")
+                raise
 
-            for cnt, batch in enumerate(self.allBatches):
-                # print(f"Posting batch {cnt + 1}/{len(self.allBatches)} for eval '{eval}'")
+        time.sleep(3)
+        listener_thread.join()
+
+        rawResults = self.socket.getReceivedData()
+        expectedRowIDs = set(rowIdMapping.keys())
+        receivedRowIDs = {key for item in rawResults for key in item.keys()}
+        missingRowIDs = list(expectedRowIDs - receivedRowIDs)
+
+        if missingRowIDs:
+            dataFromDb = fetchData(workspaceID, activePlayground, missingRowIDs)
+            rawResults.extend(dataFromDb)
+
+        working_df["context"] = None
+        for item in rawResults:
+            for compound_key, value in item.items():
+                if compound_key in rowIdMapping:
+                    idx = rowIdMapping[compound_key]["index"]
+                    working_df.at[idx, "context"] = value.get("value")
+
+        # Output generation
+        if generateOutput == True:
+            working_df = self._outputForStream(working_df, modelAliases, prompt_template, apiKey)
+
+        # Optional evaluation
+        outputEvalMapping = None
+        if toEvaluate:
+            for evalName in evals:
+
+                # Validate API and dependencies
+                self.validateApiKey(evalName=evalName)
+                metricDependencies = checkDependency(
+                    evalName, list(working_df.columns), tocheck=True
+                )
+                if not metricDependencies["status"]:
+                    raise LlumoAIError.dependencyError(metricDependencies["message"])
+
+            working_df, outputEvalMapping = self._evaluateForStream(working_df, evals, modelAliases, prompt_template)
+
+
+        self.socket.disconnect()
+
+        # Create experiment if required
+        if createExperiment:
+            df = working_df.fillna("Some error occured").astype(object)
+            if createPlayground(
+                    email, workspaceID, df,
+                    queryColName=queryColName,
+                    dataStreamName=streamId,
+                    promptText=prompt_template,
+                    definationMapping=self.definationMapping,
+                    evalOutputMap=outputEvalMapping
+            ):
+                print(
+                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results.")
+        else:
+            self.latestDataframe = working_df
+            return working_df
+
+    def _outputForStream(self, df, modelAliases, prompt_template, apiKey):
+        executor = ModelExecutor(apiKey)
+
+        for indx, row in df.iterrows():
+            inputVariables = re.findall(r"{{(.*?)}}", prompt_template)
+            if not all([k in df.columns for k in inputVariables]):
+                raise LlumoAIError.InvalidPromptTemplate()
+
+            inputDict = {key: row[key] for key in inputVariables}
+            for i, model in enumerate(modelAliases, 1):
                 try:
-                    self.postDataStream(batch=batch, workspaceID=workspaceID)
-                    # print(f"Batch {cnt + 1} posted successfully")
+                    provider = getProviderFromModel(model)
+                    filled_template = getInputPopulatedPrompt(prompt_template, inputDict)
+                    response = executor.execute(provider, model.value, filled_template, apiKey)
+                    df.at[indx, f"output_{i}"] = response
                 except Exception as e:
-                    print(f"Error posting batch {cnt + 1}: {str(e)}")
-                    continue
+                    df.at[indx, f"output_{i}"] = str(e)
+        return df
 
-                # Small delay between batches to prevent overwhelming the server
-                time.sleep(1)
+    def _evaluateForStream(self, df, evals, modelAliases, prompt_template):
+        dfWithEvals = df.copy()
 
-            # updating the dict for row column mapping
-            self.AllProcessMapping()
-            # Calculate a reasonable timeout based on the data size
-            timeout = max(60, min(600, total_items * 10))
-            # print(f"All batches posted. Waiting up to {timeout} seconds for results...")
+        outputColMapping = {}
 
-            # Listen for results
-            self.socket.listenForResults(
-                min_wait=20,
-                max_wait=timeout,
-                inactivity_timeout=30,
-                expected_results=None,
-            )
+        for i, model in enumerate(modelAliases, 1):
+            outputColName = f"output_{i}"
+            try:
 
-            # Get results for this evaluation
-            eval_results = self.socket.getReceivedData()
-            # print(f"Received {len(eval_results)} results for evaluation '{eval}'")
+                res = self.evaluateMultiple(
+                    dfWithEvals.to_dict("records"),
+                    evals=evals,
+                    prompt_template=prompt_template,
+                    outputColName=outputColName,
+                    _tocheck=False,
+                )
+                for evalMetric in evals:
+                    scoreCol = f"{evalMetric}"
+                    reasonCol = f"{evalMetric} Reason"
 
-            # Add these results to our overall results
-            results["Data Stream"] = self.finalResp(eval_results)
-            print(f"=======You are all set! continue your expectations 🚀======\n")
+                    if scoreCol in res.columns:
+                        res = res.rename(columns={scoreCol: f"{scoreCol}_{i}"})
+                    if reasonCol in res.columns:
+                        res = res.rename(columns={reasonCol: f"{evalMetric}_{i} Reason"})
 
-            # print("All evaluations completed successfully")
+                    outputColMapping[f"{scoreCol}_{i}"] = outputColName
 
-        except Exception as e:
-            print(f"Error during evaluation: {e}")
-            raise
-        finally:
-            # Always disconnect the socket when done
+                newCols = [col for col in res.columns if col not in dfWithEvals.columns]
+                dfWithEvals = pd.concat([dfWithEvals, res[newCols]], axis=1)
+            except Exception as e:
+                print(f"Evaluation failed for model {model.value}: {str(e)}")
+        return dfWithEvals, outputColMapping
+
+    def runDataStream(
+            self,
+            data,
+            streamName: str,
+            queryColName: str = "query",
+            createExperiment: bool = False,
+           ):
+
+
+        # Copy the original dataframe
+        original_df = pd.DataFrame(data)
+        working_df = original_df.copy()
+
+        # Connect to socket
+        self.socket = LlumoSocketClient(socketUrl)
+        socketID = self.socket.connect(timeout=150)
+        waited_secs = 0
+        while not self.socket._connection_established.is_set():
+            time.sleep(0.1)
+            waited_secs += 0.1
+            if waited_secs >= 20:
+                raise RuntimeError("Timeout waiting for server 'connection-established' event.")
+
+        self.validateApiKey()
+
+        # Check user credits
+        userHits = checkUserHits(
+            self.workspaceID, self.hasSubscribed, self.trialEndDate,
+            self.subscriptionEndDate, self.hitsAvailable, len(working_df)
+        )
+        if not userHits["success"]:
+            raise LlumoAIError.InsufficientCredits(userHits["message"])
+
+        print("====🚀Sit back while we fetch data from the stream 🚀====")
+        workspaceID, email = self.workspaceID, self.email
+        activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
+        streamId = getStreamId(workspaceID, self.apiKey, streamName)
+
+        # Prepare batches
+        rowIdMapping = {}
+        self.allBatches = []
+        currentBatch = []
+
+        expectedResults = len(working_df)
+        timeout = max(100, min(150, expectedResults * 10))
+
+        listener_thread = threading.Thread(
+            target=self.socket.listenForResults,
+            kwargs={
+                "min_wait": 40,
+                "max_wait": timeout,
+                "inactivity_timeout": 10,
+                "expected_results": expectedResults,
+            },
+            daemon=True
+        )
+        listener_thread.start()
+
+        for index, row in working_df.iterrows():
+            rowID, columnID = uuid.uuid4().hex, uuid.uuid4().hex
+            compoundKey = f"{rowID}-{columnID}-{columnID}"
+            rowIdMapping[compoundKey] = {"index": index}
+            templateData = {
+                "processID": getProcessID(),
+                "socketID": socketID,
+                "processData": {
+                    "executionDependency": {"query": row[queryColName]},
+                    "dataStreamID": streamId,
+                },
+                "workspaceID": workspaceID,
+                "email": email,
+                "type": "DATA_STREAM",
+                "playgroundID": activePlayground,
+                "processType": "DATA_STREAM",
+                "rowID": rowID,
+                "columnID": columnID,
+                "source": "SDK",
+            }
+            currentBatch.append(templateData)
+            if len(currentBatch) == 10 or index == len(working_df) - 1:
+                self.allBatches.append(currentBatch)
+                currentBatch = []
+
+        for batch in tqdm(self.allBatches, desc="Processing Batches", unit="batch", colour="magenta", ncols=80):
             try:
-                self.socket.disconnect()
-                # print("Socket disconnected")
+                self.postDataStream(batch=batch, workspaceID=workspaceID)
+                time.sleep(3)
             except Exception as e:
-                print(f"Error disconnecting socket: {e}")
+                print(f"Error posting batch: {e}")
+                raise
 
-        for streamName, records in results.items():
-            dataframe[streamName] = None
-            for item in records:
-                for compound_key, value in item.items():
-                    # for compound_key, value in item['data'].items():
+        time.sleep(3)
+        listener_thread.join()
 
-                    rowID = compound_key.split("-")[0]
-                    # looking for the index of each rowID , in the original dataframe
-                    if rowID in rowIdMapping:
-                        index = rowIdMapping[rowID]
-                        # dataframe.at[index, evalName] = value
-                        dataframe.at[index, streamName] = value["value"]
+        rawResults = self.socket.getReceivedData()
+        expectedRowIDs = set(rowIdMapping.keys())
+        receivedRowIDs = {key for item in rawResults for key in item.keys()}
+        missingRowIDs = list(expectedRowIDs - receivedRowIDs)
 
-                    else:
-                        pass
-                        # print(f"⚠️ Warning: Could not find rowID {rowID} in mapping")
+        if missingRowIDs:
+            dataFromDb = fetchData(workspaceID, activePlayground, missingRowIDs)
+            rawResults.extend(dataFromDb)
 
-        if createExperiment:
-            pd.set_option("future.no_silent_downcasting", True)
-            df = dataframe.fillna("Some error occured").astype(object)
+        working_df["context"] = None
+        for item in rawResults:
+            for compound_key, value in item.items():
+                if compound_key in rowIdMapping:
+                    idx = rowIdMapping[compound_key]["index"]
+                    working_df.at[idx, "context"] = value.get("value")
 
+
+
+        self.socket.disconnect()
+
+        # Create experiment if required
+        if createExperiment:
+            df = working_df.fillna("Some error occured").astype(object)
             if createPlayground(
-                email,
-                workspaceID,
-                df,
-                queryColName=queryColName,
-                dataStreamName=streamId,
+                    email, workspaceID, df,
+                    queryColName=queryColName,
+                    dataStreamName=streamId,
+                    definationMapping=self.definationMapping,
             ):
                 print(
-                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
-                )
+                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results.")
         else:
-            self.latestDataframe = dataframe
-            return dataframe
+            self.latestDataframe = working_df
+            return working_df
+
 
     def createExperiment(self, dataframe):
         try:
@@ -1165,18 +1376,8 @@ class LlumoClient:
 
         workspaceID = None
         email = None
-        socketID = self.socket.connect(timeout=150)
-        self.allBatches = []
-
-        # Wait for socket connection
-        max_wait_secs = 20
-        waited_secs = 0
-        while not self.socket._connection_established.is_set():
-            time.sleep(0.1)
-            waited_secs += 0.1
-            if waited_secs >= max_wait_secs:
-                raise RuntimeError("Timeout waiting for server connection")
 
+     
         try:
             self.validateApiKey()
         except Exception as e:
@@ -1195,14 +1396,21 @@ class LlumoClient:
             elif ext in [".xlsx", ".xls"]:
                 df = pd.read_excel(file_path)
             elif ext == ".json":
-                df = pd.read_json(file_path)
+                df = pd.read_json(file_path, orient="records")
             elif ext == ".parquet":
                 df = pd.read_parquet(file_path)
             else:
                 raise ValueError(f"Unsupported file format: {ext}")
 
             # If successfully loaded, call createPlayground
-            createPlayground(self.email, self.workspaceID, df, _uploadViaSDK= True)
+            df = df.astype(str)
+            if createPlayground(self.email, self.workspaceID, df):
+                
+                print(
+                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
+                )
+                
+                return True
 
         except Exception as e:
             print(f"Error: {e}")

llumo/exceptions.py

@@ -50,6 +50,10 @@ class LlumoAIError(Exception):
     def dependencyError(details):
         return LlumoAIError(details)
 
+    @staticmethod
+    def providerError(details):
+        return LlumoAIError(details)
+
     # @staticmethod
     # def dateNotFound():
     #     return LlumoAIError("Trial end date or subscription end date not found for the given user.")

llumo/helpingFuntions.py

@@ -9,6 +9,8 @@ import base64
 import os
 import re
 
+
+from .models import  _MODEL_METADATA,  AVAILABLEMODELS
 subscriptionUrl = "https://app.llumo.ai/api/workspace/record-extra-usage"
 getStreamdataUrl = "https://app.llumo.ai/api/data-stream/all"
 createPlayUrl = "https://app.llumo.ai/api/New-Eval-API/create-new-eval-playground"
@@ -212,8 +214,8 @@ def deleteColumnListInPlayground(workspaceID: str, playgroundID: str):
         print("❌ Error:", response.status_code, response.text)
         return None
 
-
-def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColName=None,outputColName= "output",dataStreamName=None,definationMapping=None,uploadViaSDK = False):
+def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColName=None,
+                 outputColName= "output",dataStreamName=None,definationMapping=None,evalOutputMap = None):
     if len(dataframe) > 100:
         dataframe = dataframe.head(100)
         print("⚠️ Dataframe truncated to 100 rows for upload.")
@@ -233,12 +235,12 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
     # Iterate over each column in the dataframe
     for indx, col in enumerate(dataframe.columns):
         # Generate a unique column ID using uuid
-        columnID = str(uuid.uuid4().hex[:8]) 
+        columnID = str(uuid.uuid4().hex[:8])
 
         columnIDMapping[col] = columnID
 
-        
-        if col.startswith('output') and uploadViaSDK == False:
+
+        if col.startswith('output') and promptText!=None:
             # For output columns, create the prompt template with promptText
             if promptText:
                 # Extract variables from promptText and set them as dependencies
@@ -249,12 +251,12 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
 
                 # Loop through each variable and check if it exists as a column name
                 for var in variables:
-                    varName = var.strip() 
+                    varName = var.strip()
                     if varName in columnIDMapping:  # Check if the variable is a column name
                         dependencies.append(columnIDMapping[varName])  # Add its columnID
 
                 # Now update the template for the output column
-                
+
                 template={
                     "provider": "OPENAI",
                     "model": "GPT_4o",
@@ -276,8 +278,8 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
                     "type": "PROMPT",
                     "order": indx,
                 }
-                
-        elif col.startswith('Data ') :
+
+        elif col.startswith('context') and dataStreamName != None :
             if queryColName and dataStreamName:
                 dependencies = []
                 dependencies.append(columnIDMapping[queryColName])
@@ -287,22 +289,27 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
                     "dataStreamName": dataStreamName,
                     "query": columnIDMapping[queryColName],
                     "columnID": columnID,  # Use the generated column ID
-                    "label": "Data stream",
+                    "label": "context",
                     "type": "DATA_STREAM",
                     "order": indx}
 
-        elif col in allEvals and uploadViaSDK == False:
 
+        elif any(col.startswith(eval + "_") or col == eval for eval in allEvals) and not " Reason" in col and promptText is not None:
+            if evalOutputMap != None:
+                outputColName = evalOutputMap[col]
+            else:
+                outputColName = outputColName
             dependencies = []
             variables = re.findall(r'{{(.*?)}}', promptText)
 
                 # Loop through each variable and check if it exists as a column name
             for var in variables:
-                varName = var.strip() 
+                varName = var.strip()
                 if varName in columnIDMapping:  # Check if the variable is a column name
                     dependencies.append(columnIDMapping[varName])
-                    
+
             dependencies.append(columnIDMapping[outputColName])  # Add the output column ID
+
             longDef = definationMapping.get(col, {}).get('definition', "")
             shortDef =definationMapping.get(col, {}).get('briefDefinition', "")
             enum =  col.upper().replace(" ","_")
@@ -341,12 +348,12 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
       "order": indx
     }
 
-        elif col.endswith(' Reason'):
-            continue   
+        elif col.endswith(' Reason') and promptText!=None:
+            continue
+
 
-            
         else:
-            
+
             template = {
                 "label": col,  # Label is the column name
                 "type": "VARIABLE",  # Default type for non-output columns
@@ -371,25 +378,27 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
         row_dict = {}
 
         # For each column, we need to map the column ID to the corresponding value in the row
+        print(dataframe.columns)
         for col in dataframe.columns:
             columnID = columnIDMapping[col]
-            
-            if col in allEvals:
+
+            if any(col.startswith(eval + "_") or col == eval for eval in allEvals) and not " Reason"  in col and promptText!=None:
+                print(col)
                 row_dict[columnID] = {
-                    
+
                     "value": row[col],
                     "type": "EVAL",
                     "isValid": True,
                     "reasoning": row[col+" Reason"],
                     "edgeCase": "minorHallucinationDetailNotInContext",
                     "kpi": col
-                
-                    }
-            elif col.endswith(' Reason'):
+
+                }
+            elif col.endswith(' Reason') and promptText!=None:
                 continue
             else:# Get the columnID from the mapping
                 row_dict[columnID] = row[col]
-            
+
             # row_dict[columnID] = row[col]  # Directly map the column ID to the row value
         # Add the row index (if necessary)
         row_dict["pIndex"] = indx
@@ -397,6 +406,7 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
 
     # Return the column template, row template, and the column ID mapping
     return coltemplate, rowTemplate
+
 def uploadColumnListInPlayground(payload):
     url = uploadColList
     headers = {
@@ -440,15 +450,14 @@ def uploadRowsInDBPlayground(payload):
         return None
 
 
-def createPlayground(email, workspaceID, df, promptText=None,queryColName=None,dataStreamName=None,definationMapping=None,outputColName="output",_uploadViaSDK = False):
+def createPlayground(email, workspaceID, df, promptText=None,queryColName=None,dataStreamName=None,definationMapping=None,outputColName="output",evalOutputMap = None):
     
     playgroundId = str(createEvalPlayground(email=email, workspaceID=workspaceID))
     payload1, payload2 = createColumn(
-        workspaceID=workspaceID, dataframe=df, playgroundID=playgroundId, promptText=promptText,queryColName=queryColName,dataStreamName=dataStreamName,definationMapping=definationMapping,outputColName=outputColName,uploadViaSDK=_uploadViaSDK
+        workspaceID=workspaceID, dataframe=df, playgroundID=playgroundId, promptText=promptText,queryColName=queryColName,dataStreamName=dataStreamName,definationMapping=definationMapping,outputColName=outputColName,evalOutputMap=evalOutputMap
     )
     
-    
-    
+ # Debugging line to check the payload2 structure
     deleteExistingRows = deleteColumnListInPlayground(
         workspaceID=workspaceID, playgroundID=playgroundId
     )
@@ -460,6 +469,7 @@ def createPlayground(email, workspaceID, df, promptText=None,queryColName=None,d
 
 
 
+
 def getPlaygroundInsights(workspaceID: str, activePlayground: str):
     headers = {
         
@@ -563,4 +573,62 @@ def checkDependency(selectedEval, columns,tocheck=True):
                     }
         return {"status":True,"message":"success"}
     else:
-        return {"status":True,"message":"success"}
+        return {"status":True,"message":"success"}
+    
+
+def fetchData(workspaceID, playgroundID, missingList: list):
+    # Define the URL and prepare the payload
+    socket_data_url = "https://app.llumo.ai/api/eval/get-awaited"
+    payload = {
+        "workspaceID": workspaceID,
+        "playgroundID": playgroundID,
+        "missingList": missingList
+    }
+    
+    try:
+        # Send a POST request to the API
+        response = requests.post(socket_data_url, json=payload)
+        
+        # Check if the response is successful
+        if response.status_code == 200:
+            # Parse the JSON data from the response
+            data = response.json().get("data", {})
+            
+            
+            # Prepare the list of all data values in the desired format
+            result_list = []
+            for key, value in data.items():
+                # Create a dictionary for each item in the response data
+                result_list.append({
+                    key: {
+                        "value": value.get("value"),
+                        "reasoning": value.get("reasoning"),
+                        "edgeCase": value.get("edgeCase"),
+                        "kpi": value.get("kpi")
+                    }
+                })
+            
+            return result_list
+        else:
+            print(f"Failed to fetch data. Status Code: {response.status_code}")
+            return []
+
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        return []
+
+def validateModels(model_aliases):
+
+    selectedProviders = []
+    for name in model_aliases:
+        for alias ,(provider , modelName ) in _MODEL_METADATA.items():
+            if modelName == name:
+                selectedProviders.append(provider)
+
+    if len(set(selectedProviders)) > 1:
+        return {"status": False,"message":"All selected models should be of same provider."}
+    else:
+        return {"status": True,"message":"All selected models are of same provider."}
+
+
+

llumo/models.py

@@ -6,35 +6,72 @@ class Provider(str, Enum):
 
 # Maps model aliases → (provider, actual model name for API)
 _MODEL_METADATA = {
-    "GPT_4": (Provider.OPENAI, "gpt-4"),
-    "GPT_4_32K": (Provider.OPENAI, "gpt-4-32k"),
-    "GPT_35T": (Provider.OPENAI, "gpt-3.5-turbo"),
-    "GPT_35T_INS": (Provider.OPENAI, "gpt-3.5-turbo-instruct"),
-    "GPT_35T_16K": (Provider.OPENAI, "gpt-3.5-turbo-16k"),
-    "GPT_35_TURBO": (Provider.OPENAI, "gpt-3.5-turbo"),
-
-    "GOOGLE_15_FLASH": (Provider.GOOGLE, "gemini-1.5-flash-latest"),
-    "GEMINI_PRO": (Provider.GOOGLE, "gemini-pro"),
-    "TEXT_BISON": (Provider.GOOGLE, "text-bison-001"),
-    "CHAT_BISON": (Provider.GOOGLE, "chat-bison-001"),
-    "TEXT_BISON_32K": (Provider.GOOGLE, "text-bison-32k"),
-    "TEXT_UNICORN": (Provider.GOOGLE, "text-unicorn-experimental"),
+    "GPT_4O": (Provider.OPENAI, "GPT_4O"),
+    "GPT_4_5": (Provider.OPENAI, "GPT_4_5"),
+    "GPT_4": (Provider.OPENAI, "GPT_4"),
+    "GPT_4_32K": (Provider.OPENAI, "GPT_4_32K"),
+    "GPT_3_5_Turbo": (Provider.OPENAI, "GPT_35T"),
+    "GPT_3_5_Turbo_Instruct": (Provider.OPENAI, "GPT_35T_INS"),
+    "GPT_3_5_Turbo_16K": (Provider.OPENAI, "GPT_35T_16K"),
+    "GPT_4_o_Mini": (Provider.OPENAI, "GPT_4O_MINI"),
+    "o4_MINI": (Provider.OPENAI, "O4_MINI"),
+    "o4_MINI_HIGH": (Provider.OPENAI, "O4_MINI_HIGH"),
+    "GPT_4_1": (Provider.OPENAI, "GPT_4_1"),
+    "GPT_4_1_Mini": (Provider.OPENAI, "GPT_4_1_MINI"),
+    "GPT_4_1_nano": (Provider.OPENAI, "GPT_4_1_NANO"),
+    "o3": (Provider.OPENAI, "O3"),
+    "o3_MINI": (Provider.OPENAI, "O3_MINI"),
+    "o1": (Provider.OPENAI, "O1"),
+    "o1_MINI": (Provider.OPENAI, "O1_MINI"),
+    
+    
+    "Gemini_2_5_Pro": (Provider.GOOGLE, "GEMINI_2_5_PRO"),
+    "Gemini_2_5_Flash": (Provider.GOOGLE, "GEMINI_2_5_FLASH"),
+    "Gemini_2_0": (Provider.GOOGLE, "GEMINI_2_0"),
+    "Gemini_2_0_Flash": (Provider.GOOGLE, "GEMINI_2_0_FLASH"),
+    "Gemini_Pro": (Provider.GOOGLE, "GEMINI_PRO"),
+    "Text_Bison": (Provider.GOOGLE, "TEXT_BISON"),
+    "Chat_Bison": (Provider.GOOGLE, "CHAT_BISON"),
+    "Text_Bison_32k": (Provider.GOOGLE, "TEXT_BISON_32K"),
+    "Text_Unicorn": (Provider.GOOGLE, "TEXT_UNICORN"),
+    "Google_1_5_Flash": (Provider.GOOGLE, "GOOGLE_15_FLASH"),
+    "Gemma_3_9B": (Provider.GOOGLE, "GEMMA_3_9B"),
+    "Gemma_3_27B": (Provider.GOOGLE, "GEMMA_3_27B"),
 }
 
 class AVAILABLEMODELS(str, Enum):
-    GPT_4 = "gpt-4"
-    GPT_4_32K = "gpt-4-32k"
-    GPT_35T = "gpt-3.5-turbo"
-    GPT_35T_INS = "gpt-3.5-turbo-instruct"
-    GPT_35T_16K = "gpt-3.5-turbo-16k"
-    GPT_35_TURBO = "gpt-3.5-turbo"
-
-    GOOGLE_15_FLASH = "gemini-1.5-flash-latest"
-    GEMINI_PRO = ""
-    TEXT_BISON = "text-bison-001"
-    CHAT_BISON = "chat-bison-001"
-    TEXT_BISON_32K = "text-bison-32k"
-    TEXT_UNICORN = "text-unicorn-experimental"
+    GPT_4o= "GPT_4O",
+    GPT_4o_Mini= "GPT_4O_MINI",
+    GPT_4_5= "GPT_4_5",
+    GPT_4= "GPT_4",
+    GPT_4_32K= "GPT_4_32K",
+    GPT_3_5_Turbo= "GPT_35T",
+    GPT_3_5_Turbo_Instruct= "GPT_35T_INS",
+    GPT_3_5_Turbo_16K= "GPT_35T_16K",
+    GPT_4_o_Mini= "GPT_4O_MINI",
+    o4_MINI = "O4_MINI",
+    o4_MINI_HIGH = "O4_MINI_HIGH",
+    GPT_4_1 = "GPT_4_1",
+    GPT_4_1_Mini = "GPT_4_1_MINI",
+    GPT_4_1_nano = "GPT_4_1_NANO",
+    o3 = "O3",
+    o3_MINI = "O3_MINI",
+    o1 = "O1",
+    o1_MINI = "O1_MINI",
+    
+    Gemini_2_5_Pro = "GEMINI_2_5_PRO",
+    Gemini_2_5_Flash = "GEMINI_2_5_FLASH",
+    Gemini_2_0 = "GEMINI_2_0",
+    Gemini_2_0_Flash = "GEMINI_2_0_FLASH",
+    Gemini_Pro = "GEMINI_PRO",
+    Text_Bison = "TEXT_BISON",
+    Chat_Bison = "CHAT_BISON",
+    Text_Bison_32k = "TEXT_BISON_32K",
+    Text_Unicorn = "TEXT_UNICORN",
+    Google_1_5_Flash = "GOOGLE_15_FLASH",
+    Gemma_3_9B = "GEMMA_3_9B",
+    Gemma_3_27B = "GEMMA_3_27B",
+    
 
 def getProviderFromModel(model: AVAILABLEMODELS) -> Provider:
     for alias, (provider, apiName) in _MODEL_METADATA.items():

llumo/sockets.py

@@ -17,10 +17,10 @@ class LlumoSocketClient:
 
         # Initialize client
         self.sio = socketio.Client(
-            logger=True,
-            engineio_logger=True,
+            logger=False,
+            engineio_logger=False,
             reconnection=True,
-            reconnection_attempts=10,
+            reconnection_attempts=1,
             reconnection_delay=1,
         )
 

{llumo-0.2.14b6.dist-info → llumo-0.2.15b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.14b6
+Version: 0.2.15b1
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

llumo-0.2.15b1.dist-info/RECORD

@@ -0,0 +1,13 @@
+llumo/__init__.py,sha256=O04b4yW1BnOvcHzxWFddAKhtdBEhBNhLdb6xgnpHH_Q,205
+llumo/client.py,sha256=XljwD5mZxjyrXHhu8YhN0cGsd-O_LyKbPzrhS8zbqZo,53778
+llumo/exceptions.py,sha256=Vp_MnanHbnd1Yjuoi6WLrKiwwZbJL3znCox2URMmGU4,2032
+llumo/execution.py,sha256=x88wQV8eL99wNN5YtjFaAMCIfN1PdfQVlAZQb4vzgQ0,1413
+llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
+llumo/helpingFuntions.py,sha256=0-ZwG0fnbfP4DP1JTMewM8LdXzz_-p1gRqhPsX0Zmpk,22785
+llumo/models.py,sha256=aVEZsOOoQx5LeNtwSyBxqvrINq0izH3QWu_YjsMPE6o,2910
+llumo/sockets.py,sha256=I2JO_eNEctRo_ikgvFVp5zDd-m0VDu04IEUhhsa1Tic,5950
+llumo-0.2.15b1.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
+llumo-0.2.15b1.dist-info/METADATA,sha256=yDLkiD46Qq44PA3ylKK2dzsXZmnuE23yxH0RmoqizOk,1521
+llumo-0.2.15b1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+llumo-0.2.15b1.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
+llumo-0.2.15b1.dist-info/RECORD,,

llumo-0.2.14b6.dist-info/RECORD

@@ -1,13 +0,0 @@
-llumo/__init__.py,sha256=O04b4yW1BnOvcHzxWFddAKhtdBEhBNhLdb6xgnpHH_Q,205
-llumo/client.py,sha256=YmvbfyWR9YCDOFrKM0nwDMWYLGpu4RSZwbkiUJ3e78M,46162
-llumo/exceptions.py,sha256=i3Qv4_g7XjRuho7-b7ybjw2bwSh_NhvICR6ZAgiLQX8,1944
-llumo/execution.py,sha256=x88wQV8eL99wNN5YtjFaAMCIfN1PdfQVlAZQb4vzgQ0,1413
-llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
-llumo/helpingFuntions.py,sha256=f2Y-x-DbGk3E29qaJWDOsTkuqqDFl9-VQTRM490amE4,20443
-llumo/models.py,sha256=YH-qAMnShmUpmKE2LQAzQdpRsaXkFSlOqMxHwU4zBUI,1560
-llumo/sockets.py,sha256=-zJYRCDRwElIPr5iOFqzQxjecuLJ7mztiyYJz14pGLY,5949
-llumo-0.2.14b6.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
-llumo-0.2.14b6.dist-info/METADATA,sha256=2Yl4gnAXsfpJWLB6mhlza0HUE76uJY3sC1TWK7GlUu4,1521
-llumo-0.2.14b6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-llumo-0.2.14b6.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
-llumo-0.2.14b6.dist-info/RECORD,,