llumo 0.2.14b1__py3-none-any.whl → 0.2.14b3__py3-none-any.whl

llumo/client.py

@@ -16,10 +16,11 @@ from .exceptions import LlumoAIError
 from .helpingFuntions import *
 from .sockets import LlumoSocketClient
 from .functionCalling import LlumoAgentExecutor
+import threading
+from tqdm import tqdm
 
-
-postUrl = "https://app.llumo.ai/api/eval/run-multiple-column"
-fetchUrl = "https://app.llumo.ai/api/eval/fetch-rows-data-by-column"
+postUrl = "https://red-skull-service-392377961931.us-central1.run.app/api/process-playground"
+fetchUrl = "https://red-skull-service-392377961931.us-central1.run.app/api/get-cells-data"
 validateUrl = "https://app.llumo.ai/api/workspace-details"
 socketUrl = "https://red-skull-service-392377961931.us-central1.run.app/"
 
@@ -32,7 +33,7 @@ class LlumoClient:
         self.processMapping = {}
         self.definationMapping = {}
 
-    def validateApiKey(self, evalName=" "):
+    def validateApiKey(self, evalName="Input Bias"):
         headers = {
             "Authorization": f"Bearer {self.apiKey}",
             "Content-Type": "application/json",
@@ -43,11 +44,7 @@ class LlumoClient:
             response = requests.post(url=validateUrl, json=reqBody, headers=headers)
             
             
-            try:
-                response_preview = response.text[:500]  # First 500 chars
-                # print(f"Response preview: {response_preview}")
-            except Exception as e:
-                print(f"Could not get response preview: {e}")
+            
 
         except requests.exceptions.RequestException as e:
             print(f"Request exception: {str(e)}")
@@ -328,11 +325,11 @@ class LlumoClient:
                 try:
                     
                     self.postBatch(batch=batch, workspaceID=workspaceID)
-                    # print("Betch Posted with item len: ", len(batch))
+                    print("Betch Posted with item len: ", len(batch))
                 except Exception as e:
                     continue
 
-                time.sleep(3)
+                # time.sleep(3)
 
             timeout = max(50, min(600, totalItems * 10))
 
@@ -383,69 +380,6 @@ class LlumoClient:
             return dataframe
 
     # this function allows the users to run multiple evals at once
-    def evaluateMultiple(
-            self,
-            data,
-            eval=["Response Completeness"],
-            prompt_template="Give answer to the given query:{{query}} , using the given context: {{context}}",
-            outputColName="output",
-            createExperiment: bool = False,
-            _tocheck = True,
-    ):
-        """
-        Runs multiple evaluation metrics on the same input dataset.
-
-        Parameters:
-            data (list of dict): Input data, where each dict represents a row.
-            eval (list of str): List of evaluation metric names to run.
-            prompt_template (str): Optional prompt template used in evaluation.
-            outputColName (str): Column name in data that holds the model output.
-            createExperiment (bool): Whether to log the results to Llumo playground.
-
-        Returns:
-            pandas.DataFrame: Final dataframe with all evaluation results.
-        """
-        
-        # Convert input dict list into a DataFrame
-        dataframe = pd.DataFrame(data)
-
-        # Copy to hold final results
-        resultdf = dataframe.copy()
-
-        # Run each evaluation metric one by one
-        for evalName in eval:
-            time.sleep(2)  # small delay to avoid overload or rate limits
-
-            # Call evaluate (assumes evaluate takes dict, not dataframe)
-            resultdf = self.evaluate(
-                data=resultdf.to_dict(orient="records"),  # convert df back to dict list
-                eval=evalName,
-                prompt_template=prompt_template,
-                outputColName=outputColName,
-                createExperiment=False,
-                _tocheck=_tocheck,
-            )
-
-        # Save to playground if requested
-        if createExperiment:
-            pd.set_option("future.no_silent_downcasting", True)
-            df = resultdf.fillna("Some error occured").astype(object)
-
-            if createPlayground(
-                    self.email,
-                    self.workspaceID,
-                    df,
-                    definationMapping=self.definationMapping,
-                    outputColName=outputColName,
-                    promptText=prompt_template
-            ):
-                print(
-                    "Your data has been saved in the Llumo Experiment. "
-                    "Visit https://app.llumo.ai/evallm to see the results. "
-                    "Please rerun the experiment to see the results on playground."
-                )
-        else:
-            return resultdf
 
     def evaluateCompressor(self, data, prompt_template):
         results = []
@@ -598,22 +532,223 @@ class LlumoClient:
         # dataframe["cost_saving"] = cost_saving
 
         return dataframe
+
+    def evaluateMultiple(
+            self,
+            data,
+            evals: list,  # list of eval metric names
+            prompt_template="",
+            outputColName="output",
+            createExperiment: bool = False,
+            _tocheck=True,
+    ):
+        dataframe = pd.DataFrame(data)
+        workspaceID = None
+        email = None
+        socketID = self.socket.connect(timeout=150)
+        self.allBatches = []
+        rowIdMapping = {}  # (rowID-columnID-columnID -> (index, evalName))
+
+        # Wait for socket connection
+        max_wait_secs = 20
+        waited_secs = 0
+        while not self.socket._connection_established.is_set():
+            time.sleep(0.1)
+            waited_secs += 0.1
+            if waited_secs >= max_wait_secs:
+                raise RuntimeError("Timeout waiting for server connection")
+
+        # Start listener thread
+        expectedResults = len(dataframe) * len(evals)
+        # print("expected result" ,expectedResults)
+        timeout = max(100, min(150, expectedResults * 10))
+        listener_thread = threading.Thread(
+            target=self.socket.listenForResults,
+            kwargs={
+                "min_wait": 40,
+                "max_wait": timeout,
+                "inactivity_timeout": 200,
+                "expected_results": expectedResults,
+            },
+            daemon=True
+        )
+        listener_thread.start()
+
+        for evalName in evals:
+            # print(f"\n======= Running evaluation for: {evalName} =======")
+
+            # Validate API and dependencies
+            self.validateApiKey(evalName=evalName)
+            metricDependencies = checkDependency(evalName, list(dataframe.columns), tocheck=_tocheck)
+            if not metricDependencies["status"]:
+                raise LlumoAIError.dependencyError(metricDependencies["message"])
+
+            evalDefinition = self.evalDefinition[evalName]["definition"]
+            model = "GPT_4"
+            provider = "OPENAI"
+            evalType = "LLM"
+            workspaceID = self.workspaceID
+            email = self.email
+
+            userHits = checkUserHits(
+                self.workspaceID,
+                self.hasSubscribed,
+                self.trialEndDate,
+                self.subscriptionEndDate,
+                self.hitsAvailable,
+                len(dataframe),
+            )
+            if not userHits["success"]:
+                raise LlumoAIError.InsufficientCredits(userHits["message"])
+
+            currentBatch = []
+            for index, row in dataframe.iterrows():
+                tools = [row["tools"]] if "tools" in dataframe.columns else []
+                groundTruth = row.get("groundTruth", "")
+                messageHistory = [row["messageHistory"]] if "messageHistory" in dataframe.columns else []
+                promptTemplate = prompt_template
+                keys = re.findall(r"{{(.*?)}}", promptTemplate)
+
+                if not all([ky in dataframe.columns for ky in keys]):
+                    raise LlumoAIError.InvalidPromptTemplate()
+
+                inputDict = {key: row[key] for key in keys if key in row}
+                output = row.get(outputColName, "")
+
+                activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
+                rowID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
+                columnID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
+
+                compoundKey = f"{rowID}-{columnID}-{columnID}"
+                rowIdMapping[compoundKey] = {"index": index, "eval": evalName}
+
+                templateData = {
+                    "processID": getProcessID(),
+                    "socketID": socketID,
+                    "source": "SDK",
+                    "processData": {
+                        "executionDependency": {
+                            "query": "",
+                            "context": "",
+                            "output": output,
+                            "tools": tools,
+                            "groundTruth": groundTruth,
+                            "messageHistory": messageHistory,
+                        },
+                        "definition": evalDefinition,
+                        "model": model,
+                        "provider": provider,
+                        "analytics": evalName,
+                    },
+                    "workspaceID": workspaceID,
+                    "type": "EVAL",
+                    "evalType": evalType,
+                    "kpi": evalName,
+                    "columnID": columnID,
+                    "rowID": rowID,
+                    "playgroundID": activePlayground,
+                    "processType": "EVAL",
+                    "email": email,
+                }
+
+                query = ""
+                context = ""
+                for key, value in inputDict.items():
+                    if isinstance(value, str):
+                        length = len(value.split()) * 1.5
+                        if length > 50:
+                            context += f" {key}: {value}, "
+                        else:
+                            if promptTemplate:
+                                tempObj = {key: value}
+                                promptTemplate = getInputPopulatedPrompt(promptTemplate, tempObj)
+                            else:
+                                query += f" {key}: {value}, "
+
+                if not context.strip():
+                    for key, value in inputDict.items():
+                        context += f" {key}: {value}, "
+
+                templateData["processData"]["executionDependency"]["context"] = context.strip()
+                templateData["processData"]["executionDependency"]["query"] = query.strip()
+                if promptTemplate and not query.strip():
+                    templateData["processData"]["executionDependency"]["query"] = promptTemplate
+
+                currentBatch.append(templateData)
+                if len(currentBatch) == 10:
+                    self.allBatches.append(currentBatch)
+                    currentBatch = []
+
+            if currentBatch:
+                self.allBatches.append(currentBatch)
+
+
+        for batch in tqdm(self.allBatches, desc="Processing Batches", unit="batch",colour="magenta", ascii=False):
+            try:
+                self.postBatch(batch=batch, workspaceID=workspaceID)
+
+            except Exception as e:
+                print(f"Error posting batch: {e}")
+                raise
+
+
+        # Wait for results
+        
+        listener_thread.join()
+
+        raw_results = self.socket.getReceivedData()
+
+        
+        
+        # Fix here: keep full keys, do not split keys
+        # received_rowIDs = {key for item in raw_results for key in item.keys()}
+        # expected_rowIDs = set(rowIdMapping.keys())
+        # missing_rowIDs = expected_rowIDs - received_rowIDs
+        # print("All expected keys:", expected_rowIDs)
+        # print("All received keys:", received_rowIDs)
+        # print("Missing keys:", missing_rowIDs)
+
+        # Initialize dataframe columns for each eval
+        for eval in evals:
+            dataframe[eval] = None
+            dataframe[f"{eval} Reason"] = None
+
+        # Map results to dataframe rows
+        for item in raw_results:
+            for compound_key, value in item.items():
+                if compound_key in rowIdMapping:
+                    index = rowIdMapping[compound_key]["index"]
+                    evalName = rowIdMapping[compound_key]["eval"]
+                    dataframe.at[index, evalName] = value.get("value")
+                    dataframe.at[index, f"{evalName} Reason"] = value.get("reasoning")
+
+        self.socket.disconnect()
+
+        if createExperiment:
+            pd.set_option("future.no_silent_downcasting", True)
+            df = dataframe.fillna("Some error occured").astype(object)
+            if createPlayground(email, workspaceID, df, promptText=prompt_template,
+                                definationMapping=self.definationMapping, outputColName=outputColName):
+                print(
+                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results.")
+        else:
+            return dataframe
+
     def run_sweep(
     self,
     templates: List[str],
     dataset: Dict[str, List[str]],
     model_aliases: List[AVAILABLEMODELS],
     apiKey: str,
-    eval=["Response Correctness"],
+    evals=["Response Correctness"],
     toEvaluate: bool = False,
     createExperiment: bool = False,
 ) -> pd.DataFrame:
         
-        try:
-            self.validateApiKey()
-        except Exception as e:
-            raise Exception("Some error occurred, please check your API key")
+        
+        
 
+        self.validateApiKey(evalName="")
         workspaceID = self.workspaceID
         email = self.email
         executor = ModelExecutor(apiKey)
@@ -657,14 +792,14 @@ class LlumoClient:
                 try:
                     res = self.evaluateMultiple(
                         df.to_dict("records"),
-                        eval=eval,
+                        evals=evals,
                         prompt_template=str(templates[0]),
                         outputColName=outputColName,
                         _tocheck=False,
                     )
-
+                    
                     # Rename all new columns with _i+1 (e.g., _1, _2)
-                    for evalMetric in eval:
+                    for evalMetric in evals:
                         scoreCol = f"{evalMetric}"
                         reasonCol = f"{evalMetric} Reason"
                         if scoreCol in res.columns:
@@ -675,6 +810,8 @@ class LlumoClient:
                     # Drop duplicated columns from df (like prompt, variables, etc.)
                     newCols = [col for col in res.columns if col not in dfWithEvals.columns]
                     dfWithEvals = pd.concat([dfWithEvals, res[newCols]], axis=1)
+                    
+                    
 
                 except Exception as e:
                     print(f"Evaluation failed for model {model.value}: {str(e)}")
@@ -720,22 +857,22 @@ class LlumoClient:
             dataframe, agents, model=model, model_api_key=model_api_key
         )
         
-        
-        # evals = [
-        #     "Tool Reliability",
-        #     "Stepwise Progression",
-        #     "Tool Selection Accuracy",
-        #     "Final Task Alignment",
-        # ]
 
-        for eval in evals:
+        # for eval in evals:
             # Perform evaluation
-            toolResponseDf = self.evaluate(
-                toolResponseDf.to_dict(orient = "records"),
-                eval=eval,
-                prompt_template=prompt_template,
-                createExperiment=False,
+            # toolResponseDf = self.evaluate(
+            #     toolResponseDf.to_dict(orient = "records"),
+            #     eval=eval,
+            #     prompt_template=prompt_template,
+            #     createExperiment=False,
+            # )
+        toolResponseDf = self.evaluateMultiple(
+            toolResponseDf.to_dict(orient = "records"),
+            evals=evals,
+            prompt_template=prompt_template,
+            createExperiment=False,
             )
+
         if createExperiment:
             pd.set_option("future.no_silent_downcasting", True)
             df = toolResponseDf.fillna("Some error occured")
@@ -762,21 +899,17 @@ class LlumoClient:
                     "DataFrame must contain 'query', 'messageHistory','output' ,and 'tools' columns. Make sure the columns names are same as mentioned here."
                 )
 
-
-            # evals = [
-            #     "Tool Reliability",
-            #     "Stepwise Progression",
-            #     "Tool Selection Accuracy",
-            #     "Final Task Alignment",
-            # ]
-
             toolResponseDf = dataframe.copy()
-            for eval in evals:
-                # Perform evaluation
-                toolResponseDf = self.evaluate(
-                    toolResponseDf.to_dict(orient = "records"), eval=eval, prompt_template="Give answer for the given query: {{query}}",outputColName=outputColName
-                )
-
+            # for eval in evals:
+            #     # Perform evaluation
+            #     toolResponseDf = self.evaluate(
+            #         toolResponseDf.to_dict(orient = "records"), eval=eval, prompt_template="Give answer for the given query: {{query}}",outputColName=outputColName
+            #     )
+            toolResponseDf = self.evaluateMultiple(
+                toolResponseDf.to_dict(orient = "records"),
+                eval=evals, prompt_template="Give answer for the given query: {{query}}",
+                outputColName=outputColName
+            )
             return toolResponseDf
 
         except Exception as e:
@@ -785,13 +918,13 @@ class LlumoClient:
         
     def runDataStream(
         self,
-        dataframe,
+        data,
         streamName: str,
         queryColName: str = "query",
         createExperiment: bool = False,
     ):
         results = {}
-
+        dataframe = pd.DataFrame(data)
         try:
             socketID = self.socket.connect(timeout=150)
             # Ensure full connection before proceeding
@@ -806,16 +939,11 @@ class LlumoClient:
                     )
             # print(f"Connected with socket ID: {socketID}")
             rowIdMapping = {}
-            try:
+            
                 # print(f"Validating API key...")
-                self.validateApiKey()
+            self.validateApiKey()
                 # print(f"API key validation successful. Hits available: {self.hitsAvailable}")
-            except Exception as e:
-                print(f"Error during API key validation: {str(e)}")
-                if hasattr(e, "response") and getattr(e, "response", None) is not None:
-                    print(f"Status code: {e.response.status_code}")
-                    print(f"Response content: {e.response.text[:500]}...")
-                raise
+            
             # check for available hits and trial limit
             userHits = checkUserHits(
                 self.workspaceID,

llumo/helpingFuntions.py

@@ -341,7 +341,8 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
       "order": indx
     }
 
-           
+        elif col.endswith(' Reason'):
+            continue   
 
             
         else:
@@ -384,18 +385,18 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
                     "kpi": col
                 
                     }
+            elif col.endswith(' Reason'):
+                continue
             else:# Get the columnID from the mapping
-                row_dict[columnID] = {
-                    "value": row[col],
-                    "type":"VARIABLE"# Map the columnID to the value in the row
-                }
+                row_dict[columnID] = row[col]
+            
+            # row_dict[columnID] = row[col]  # Directly map the column ID to the row value
         # Add the row index (if necessary)
         row_dict["pIndex"] = indx
         rowTemplate["dataToUploadList"].append(row_dict)
 
     # Return the column template, row template, and the column ID mapping
     return coltemplate, rowTemplate
-
 def uploadColumnListInPlayground(payload):
     url = uploadColList
     headers = {

llumo/sockets.py

@@ -17,8 +17,8 @@ class LlumoSocketClient:
 
         # Initialize client
         self.sio = socketio.Client(
-            # logger=True,
-            # engineio_logger=True,
+            logger=False,
+            engineio_logger=False,
             reconnection=True,
             reconnection_attempts=10,
             reconnection_delay=1,
@@ -104,6 +104,7 @@ class LlumoSocketClient:
         # if not self._connected:
         #     raise RuntimeError("WebSocket is not connected. Call connect() first.")
 
+        # total records
         self._expected_results = expected_results  # NEW
         start_time = time.time()
         self._last_update_time = time.time()
@@ -128,7 +129,7 @@ class LlumoSocketClient:
                     self._listening_done.set()
                     break
 
-                time.sleep(3)
+
 
         timeout_thread = threading.Thread(target=timeout_watcher, daemon=True)
         timeout_thread.start()
@@ -136,6 +137,7 @@ class LlumoSocketClient:
 
     def getReceivedData(self):
         with self._lock:
+            # print("Total received:", len(self._received_data))  # DEBUG
             return self._received_data.copy()
 
     def disconnect(self):

{llumo-0.2.14b1.dist-info → llumo-0.2.14b3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.14b1
+Version: 0.2.14b3
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo
@@ -21,6 +21,7 @@ Requires-Dist: requests>=2.0.0
 Requires-Dist: python-socketio
 Requires-Dist: python-dotenv
 Requires-Dist: openai==1.75.0
+Requires-Dist: tqdm==4.67.1
 Requires-Dist: google-generativeai==0.8.5
 Dynamic: author
 Dynamic: author-email

llumo-0.2.14b3.dist-info/RECORD

@@ -0,0 +1,13 @@
+llumo/__init__.py,sha256=O04b4yW1BnOvcHzxWFddAKhtdBEhBNhLdb6xgnpHH_Q,205
+llumo/client.py,sha256=Zque6TDcEBFO1tQjNMsmUpYVbGdooLiwDKDYPtJ2szY,43465
+llumo/exceptions.py,sha256=i3Qv4_g7XjRuho7-b7ybjw2bwSh_NhvICR6ZAgiLQX8,1944
+llumo/execution.py,sha256=x88wQV8eL99wNN5YtjFaAMCIfN1PdfQVlAZQb4vzgQ0,1413
+llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
+llumo/helpingFuntions.py,sha256=9aFvBNPB-AyeMs6c8YpdPckNeF95thmTqAAmYC1q4oo,20320
+llumo/models.py,sha256=YH-qAMnShmUpmKE2LQAzQdpRsaXkFSlOqMxHwU4zBUI,1560
+llumo/sockets.py,sha256=Rlww5z9w_Ij99Z_VzgpF93K9gSZ5YaS4A8oAY0jS1wA,5650
+llumo-0.2.14b3.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
+llumo-0.2.14b3.dist-info/METADATA,sha256=gDpqIZ7fknvE3_2FgeWMppevOCYzGac3GOUBAU4nPoU,1521
+llumo-0.2.14b3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+llumo-0.2.14b3.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
+llumo-0.2.14b3.dist-info/RECORD,,

llumo-0.2.14b1.dist-info/RECORD

@@ -1,13 +0,0 @@
-llumo/__init__.py,sha256=O04b4yW1BnOvcHzxWFddAKhtdBEhBNhLdb6xgnpHH_Q,205
-llumo/client.py,sha256=KbvBokbeINsEv2bJ9S3Zys6GYmpA7BIsjchmYZJx5C0,38133
-llumo/exceptions.py,sha256=i3Qv4_g7XjRuho7-b7ybjw2bwSh_NhvICR6ZAgiLQX8,1944
-llumo/execution.py,sha256=x88wQV8eL99wNN5YtjFaAMCIfN1PdfQVlAZQb4vzgQ0,1413
-llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
-llumo/helpingFuntions.py,sha256=FOHDxnofS316u6fnuPDVzM-fIrTzAGkMTAjoiadlKVw,20230
-llumo/models.py,sha256=YH-qAMnShmUpmKE2LQAzQdpRsaXkFSlOqMxHwU4zBUI,1560
-llumo/sockets.py,sha256=0BCcdCaiXDR7LO_9NIYA6urtpgdmyWW2M1US67G9Eus,5583
-llumo-0.2.14b1.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
-llumo-0.2.14b1.dist-info/METADATA,sha256=TxKTceUWuxS1UkZT1vd_3yVbR98_ntHSEU1yq2HW_bg,1493
-llumo-0.2.14b1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-llumo-0.2.14b1.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
-llumo-0.2.14b1.dist-info/RECORD,,