llumo 0.2.11__py3-none-any.whl → 0.2.12__py3-none-any.whl

llumo/client.py

@@ -153,6 +153,7 @@ class LlumoClient:
 
         return uniqueResults
 
+    # this function allows the users to run exactl one eval at a time
     def evaluate(
         self,
         dataframe,
@@ -369,31 +370,46 @@ class LlumoClient:
         else:
             return dataframe
 
+    # this function allows the users to run multiple evals at once
+    def evaluateMultiple(
+            self,
+            dataframe,
+            eval=["Response Completeness"],
+            prompt_template="",
+            outputColName="output",
+            createExperiment: bool = False,
+    ):
+        resultdf  = dataframe.copy()
+        for evalName in eval:
+            resultdf = self.evaluate(dataframe = resultdf,eval=evalName,prompt_template=prompt_template,outputColName=outputColName,createExperiment = False)
+
+        if createExperiment:
+            pd.set_option("future.no_silent_downcasting", True)
+            df = resultdf.fillna("Some error occured").astype(object)
+
+            if createPlayground(self.email, self.workspaceID, df):
+                print(
+                    "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
+                )
+        else:
+            return resultdf
+
+
     def evaluateCompressor(self, dataframe, prompt_template):
         results = []
 
         try:
-            # Connect to socket first
-            # print("Connecting to socket server...")
             socketID = self.socket.connect(timeout=150)
-
-            # Ensure full connection before proceeding
             max_wait_secs = 20
             waited_secs = 0
             while not self.socket._connection_established.is_set():
                 time.sleep(0.1)
                 waited_secs += 0.1
                 if waited_secs >= max_wait_secs:
-                    raise RuntimeError(
-                        "Timeout waiting for server 'connection-established' event."
-                    )
-
-            # print(f"Connected with socket ID: {socketID}")
+                    raise RuntimeError("Timeout waiting for server 'connection-established' event.")
 
             try:
-                # print(f"Validating API key...")
                 self.validateApiKey()
-                # print(f"API key validation successful. Hits available: {self.hitsAvailable}")
             except Exception as e:
                 print(f"Error during API key validation: {str(e)}")
                 if hasattr(e, "response") and getattr(e, "response", None) is not None:
@@ -401,61 +417,42 @@ class LlumoClient:
                     print(f"Response content: {e.response.text[:500]}...")
                 raise
 
-            # check for available hits and trial limit
-            userHits = checkUserHits(
-                self.workspaceID,
-                self.hasSubscribed,
-                self.trialEndDate,
-                self.subscriptionEndDate,
-                self.hitsAvailable,
-                len(dataframe),
-            )
+            userHits = checkUserHits(self.workspaceID, self.hasSubscribed, self.trialEndDate, self.subscriptionEndDate,
+                                     self.hitsAvailable, len(dataframe))
 
-            # do not proceed if subscription or trial limit has exhausted
             if not userHits["success"]:
                 raise LlumoAIError.InsufficientCredits(userHits["message"])
 
-            # if self.hitsAvailable == 0 or len(dataframe) > self.hitsAvailable:
-            #     raise LlumoAIError.InsufficientCredits()
-
             model = "GPT_4"
             provider = "OPENAI"
             evalType = "LLUMO"
             workspaceID = self.workspaceID
             email = self.email
-            # Prepare all batches before sending
-            # print("Preparing batches...")
             self.allBatches = []
             currentBatch = []
-
+            rowIdMapping = {}
             for index, row in dataframe.iterrows():
                 promptTemplate = prompt_template
-
-                # extracting the placeholders from the prompt template
                 keys = re.findall(r"{{(.*?)}}", promptTemplate)
                 inputDict = {key: row[key] for key in keys if key in row}
 
                 if not all([ky in dataframe.columns for ky in keys]):
                     raise LlumoAIError.InvalidPromptTemplate()
 
-                activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
-                    "-", ""
-                )
+                activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
                 rowID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
                 columnID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
 
-                compressed_prompt_id = (
-                    f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
-                )
-                compressed_prompt_output_id = (
-                    f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
-                )
+                compressed_prompt_id = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
+                compressed_prompt_output_id = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
                 cost_id = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
-                cost_saving_id = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
-                    "-", ""
-                )
+                cost_saving_id = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
+
+                rowDataDict = {}
+                for col in dataframe.columns:
+                    val = row[col]
+                    rowDataDict[col] = {"type": "VARIABLE", "value": str(val)}
 
-                # Use the server-provided socket ID here
                 templateData = {
                     "processID": getProcessID(),
                     "socketID": socketID,
@@ -469,13 +466,10 @@ class LlumoClient:
                         "compressed_prompt": compressed_prompt_id,
                         "compressed_prompt_output": compressed_prompt_output_id,
                         "cost": cost_id,
-                        "cost_saving": cost_saving_id,
+                        "cost_saving": cost_saving_id
                     },
                     "processData": {
-                        "rowData": {
-                            "query": {"type": "VARIABLE", "value": ""},
-                            "context": {"type": "VARIABLE", "value": ""},
-                        },
+                        "rowData": rowDataDict,
                         "dependency": list(inputDict.keys()),
                         "dependencyMapping": {ky: ky for ky in list(inputDict.keys())},
                         "provider": provider,
@@ -485,111 +479,71 @@ class LlumoClient:
                             "compressed_prompt": compressed_prompt_id,
                             "compressed_prompt_output": compressed_prompt_output_id,
                             "cost": cost_id,
-                            "cost_saving": cost_saving_id,
-                        },
+                            "cost_saving": cost_saving_id
+                        }
                     },
                     "workspaceID": workspaceID,
                     "email": email,
-                    "playgroundID": activePlayground,
+                    "playgroundID": activePlayground
                 }
 
-                # Build query/context from input
-                query = ""
-                context = ""
-
-                for key, value in inputDict.items():
-                    if isinstance(value, str):
-                        length = len(value.split()) * 1.5
-                        if length > 50:
-                            context += f" {key}: {value}, "
-                        else:
-                            if promptTemplate:
-                                populatedPrompt = getInputPopulatedPrompt(
-                                    promptTemplate, {key: value}
-                                )
-                                query += f"{populatedPrompt} "
-                            else:
-                                query += f" {key}: {value}, "
-
-                if not context.strip():
-                    for key, value in inputDict.items():
-                        context += f" {key}: {value}, "
-
-                templateData["processData"]["rowData"]["context"][
-                    "value"
-                ] = context.strip()
-                templateData["processData"]["rowData"]["query"]["value"] = query.strip()
-
-                if promptTemplate and not query.strip():
-                    templateData["processData"]["rowData"]["query"][
-                        "value"
-                    ] = promptTemplate
-
+                rowIdMapping[rowID] = index
+                # print("__________________________TEMPLATE__________________________________")
                 # print(templateData)
+
                 currentBatch.append(templateData)
 
-            if len(currentBatch) == 10 or index == len(dataframe) - 1:
-                self.allBatches.append(currentBatch)
-                currentBatch = []
+                if len(currentBatch) == 10 or index == len(dataframe) - 1:
+                    self.allBatches.append(currentBatch)
+                    currentBatch = []
 
-            # Post all batches
             total_items = sum(len(batch) for batch in self.allBatches)
-            # print(f"Posting {len(self.allBatches)} batches ({total_items} items total)")
 
             for cnt, batch in enumerate(self.allBatches):
-                # print(f"Posting batch {cnt + 1}/{len(self.allBatches)} for eval '{eval}'")
                 try:
                     self.postBatch(batch=batch, workspaceID=workspaceID)
-                    # print(f"Batch {cnt + 1} posted successfully")
                 except Exception as e:
                     print(f"Error posting batch {cnt + 1}: {str(e)}")
                     continue
-
-                # Small delay between batches to prevent overwhelming the server
                 time.sleep(1)
 
-            # updating the dict for row column mapping
             self.AllProcessMapping()
-            # Calculate a reasonable timeout based on the data size
             timeout = max(60, min(600, total_items * 10))
-            # print(f"All batches posted. Waiting up to {timeout} seconds for results...")
-
-            # Listen for results
-            self.socket.listenForResults(
-                min_wait=20,
-                max_wait=timeout,
-                inactivity_timeout=30,
-                expected_results=None,
-            )
+            self.socket.listenForResults(min_wait=20, max_wait=timeout, inactivity_timeout=30, expected_results=None)
 
-            # Get results for this evaluation
-            eval_results = self.socket.getReceivedData()
-            # print(f"Received {len(eval_results)} results for evaluation '{eval}'")
-
-            # Add these results to our overall results
-            results = self.finalResp(eval_results)
-            print(f"======= Completed evaluation: {eval} =======\n")
-
-            # print("All evaluations completed successfully")
+            results = self.socket.getReceivedData()
+            # results = self.finalResp(eval_results)
+            # print(f"======= Completed evaluation: {eval} =======\n")
 
         except Exception as e:
             print(f"Error during evaluation: {e}")
             raise
         finally:
-            # Always disconnect the socket when done
             try:
                 self.socket.disconnect()
-                # print("Socket disconnected")
             except Exception as e:
                 print(f"Error disconnecting socket: {e}")
 
-        compressed_prompt, compressed_prompt_output, cost, cost_saving = (
-            costColumnMapping(results, self.processMapping)
-        )
-        dataframe["compressed_prompt"] = compressed_prompt
-        dataframe["compressed_prompt_output"] = compressed_prompt_output
-        dataframe["cost"] = cost
-        dataframe["cost_saving"] = cost_saving
+        dataframe["Compressed Input"] = None
+        for records in results:
+            for compound_key, value in records.items():
+                # for compound_key, value in item['data'].items():
+                rowID = compound_key.split('-')[0]
+                # looking for the index of each rowID , in the original dataframe
+                if rowID in rowIdMapping:
+                    index = rowIdMapping[rowID]
+
+                    dataframe.at[index, "Compressed Input"] = value["value"]
+
+                else:
+                    pass
+                    # print(f"⚠️ Warning: Could not find rowID {rowID} in mapping")
+
+        # compressed_prompt, compressed_prompt_output, cost, cost_saving = costColumnMapping(results, self.processMapping)
+        # dataframe["compressed_prompt"] = compressed_prompt
+        # dataframe["compressed_prompt_output"] = compressed_prompt_output
+        # dataframe["cost"] = cost
+        # dataframe["cost_saving"] = cost_saving
 
         return dataframe
 
@@ -644,7 +598,7 @@ class LlumoClient:
         df = pd.DataFrame(results)
         if toEvaluate:
 
-            res = self.evaluate(df, eval=eval, prompt_template=str(templates[0]))
+            res = self.evaluateMultiple(df, eval=eval, prompt_template=str(templates[0]))
 
             if createExperiment:
                 pd.set_option("future.no_silent_downcasting", True)
@@ -667,6 +621,7 @@ class LlumoClient:
             else:
                 return df
 
+    # this function generates an output using llm and tools and evaluate that output
     def evaluateAgents(
         self,
         dataframe,
@@ -708,6 +663,7 @@ class LlumoClient:
         else:
             return toolResponseDf
 
+    # this function evaluate that tools output given by the user
     def evaluateAgentResponses(
         self,
         dataframe,

{llumo-0.2.11.dist-info → llumo-0.2.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.11
+Version: 0.2.12
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

{llumo-0.2.11.dist-info → llumo-0.2.12.dist-info}/RECORD

@@ -1,13 +1,13 @@
 llumo/__init__.py,sha256=O04b4yW1BnOvcHzxWFddAKhtdBEhBNhLdb6xgnpHH_Q,205
-llumo/client.py,sha256=_TpTvzt5ox5MUCZ5O07AY6DYrtIhtezEDAE6nG7nUjs,35632
+llumo/client.py,sha256=QMasCjzRmL38RuMUHoY39_ge2nIRdSOECjj1daicd-k,34144
 llumo/exceptions.py,sha256=iCj7HhtO_ckC2EaVBdXbAudNpuMDsYmmMEV5lwynZ-E,1854
 llumo/execution.py,sha256=x88wQV8eL99wNN5YtjFaAMCIfN1PdfQVlAZQb4vzgQ0,1413
 llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
 llumo/helpingFuntions.py,sha256=lG_d3lQgJj6pI7v1YdLqdPojrLCNwybKz29zXrGaL5k,9090
 llumo/models.py,sha256=YH-qAMnShmUpmKE2LQAzQdpRsaXkFSlOqMxHwU4zBUI,1560
 llumo/sockets.py,sha256=Qxxqtx3Hg07HLhA4QfcipK1ChiOYhHZBu02iA6MfYlQ,5579
-llumo-0.2.11.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
-llumo-0.2.11.dist-info/METADATA,sha256=ymWr0L9PnYitu0FnBAZ2NsJnlOxhcsUoUVA8Yn-DXoQ,1491
-llumo-0.2.11.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-llumo-0.2.11.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
-llumo-0.2.11.dist-info/RECORD,,
+llumo-0.2.12.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
+llumo-0.2.12.dist-info/METADATA,sha256=CYu8qMtD-XZuoQzC7YPZRlNgetEQPjMT1RiMGnHdZJs,1491
+llumo-0.2.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+llumo-0.2.12.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
+llumo-0.2.12.dist-info/RECORD,,

{llumo-0.2.11.dist-info → llumo-0.2.12.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any