llumo 0.2.32__tar.gz → 0.2.33__tar.gz

{llumo-0.2.32/llumo.egg-info → llumo-0.2.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.32
+Version: 0.2.33
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

{llumo-0.2.32 → llumo-0.2.33}/llumo/client.py

@@ -22,16 +22,22 @@ from tqdm import tqdm
 
 pd.set_option("future.no_silent_downcasting", True)
 
-postUrl = (
-    "https://redskull.llumo.ai/api/process-playground"
-)
-fetchUrl = (
-    "https://redskull.llumo.ai/api/get-cells-data"
-)
-socketDataUrl = "https://app.llumo.ai/api/eval/get-awaited"
+# postUrl = (
+#     "https://redskull.llumo.ai/api/process-playground"
+# )
+postUrl = "http://localhost:4747/api/process-playground"
+# fetchUrl = (
+#     "https://redskull.llumo.ai/api/get-cells-data"
+# )
+fetchUrl = "http://localhost:4747/api/get-cells-data"
+fetchMissingEvalUrl = "http://localhost:4747/api/get-missing-keys"
+socketDataUrl = "http://localhost:4747/api/eval/get-awaited"
 
 validateUrl = "https://app.llumo.ai/api/workspace-details"
-socketUrl = "https://redskull.llumo.ai/"
+# socketUrl = "https://redskull.llumo.ai/"
+socketUrl = "http://localhost:4747/"
+createEvalUrl = "https://backend-api.llumo.ai/api/v1/create-debug-log-for-sdk"
+# createEvalUrl = "http://localhost:4545/api/v1/create-debug-log-for-sdk"
 
 
 class LlumoClient:
@@ -200,15 +206,87 @@ class LlumoClient:
             "Content-Type": "application/json",
         }
         try:
-            # print(postUrl)
+            print(postUrl)
             response = requests.post(postUrl, json=payload, headers=headers)
-            # print(f"Post API Status Code: {response.status_code}")
-            # print(response.text)
+            print(f"Post API Status Code: {response.status_code}")
+            print(response.text)
             # print(response.status_code)
 
         except Exception as e:
             print(f"Error in posting batch: {e}")
 
+    
+    def fetchDataForMissingKeys(self, workspaceID, missingKeys: list):
+    # Define the URL and prepare the payload
+       
+        payload = {
+            "workspaceID": workspaceID,
+            "missingKeys": missingKeys
+        }
+
+        headers = {
+            "Authorization": f"Bearer {self.socketToken}",
+            "Content-Type": "application/json",
+        }
+        
+        try:
+            # Send a POST request to the API
+            response = requests.post(fetchMissingEvalUrl, json=payload, headers=headers)
+            
+            # Check if the response is successful
+            if response.status_code == 200:
+                # Parse the JSON data from the response
+                data = response.json().get("data", {})
+                
+                
+                # Prepare the list of all data values in the desired format
+                result_list = []
+                for key, value in data.items():
+                    # Create a dictionary for each item in the response data
+                    result_list.append({
+                        key: {
+                            "value": value.get("value"),
+                            "fullEval": value.get("fullEval", {}),
+                            "runLog": value.get("runLog", {}),
+                            "evallist": value.get("evallist", [])
+                        }
+                    })
+                
+                print("Fetched data for missing keys:", result_list)
+                return result_list
+            else:
+                print(f"Failed to fetch data. Status Code: {response.status_code}")
+                return []
+
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            return []
+
+    
+    def createRunForEvalMultiple(self, smartLog):
+        payload = {
+            "log": smartLog
+        }
+        # socketToken here if the "JWD" token
+        headers = {
+            # "Authorization": f"Bearer {self.socketToken}",
+            "Content-Type": "application/json",
+        }
+        try:
+            # print(postUrl)
+            print(createEvalUrl)
+            print(payload)
+            print(headers)
+            response = requests.post(createEvalUrl, json=payload, headers=headers)
+
+            print(f"Post API Status Code: {response.status_code}")
+            print(response.text)
+            # print(response.status_code)
+
+        except Exception as e:
+            print(f"Error in posting batch: {e}")
+
+
     def postDataStream(self, batch, workspaceID):
         payload = {
             "batch": json.dumps(batch),
@@ -700,8 +778,9 @@ class LlumoClient:
     def evaluateMultiple(
         self,
         data,
-        evals: list,
-        prompt_template="Give answer to the given query: {{query}} using the given context: {{context}}.",
+        evals: list = [],
+        # prompt_template="Give answer to the given query: {{query}} using the given context: {{context}}.",
+        prompt_template="",
         getDataFrame: bool = False,
         _tocheck=True,
     ):
@@ -718,8 +797,10 @@ class LlumoClient:
         email = None
         try:
             socketID = self.socket.connect(timeout=250)
+            print("Socket connected with ID:", socketID)
         except Exception as e:
             socketID = "DummySocketID"
+            print(f"Socket connection failed, using dummy ID. Error: {str(e)}")
 
         self.evalData = []
         self.evals = evals
@@ -736,27 +817,33 @@ class LlumoClient:
         #         raise RuntimeError("Timeout waiting for server connection")
 
         # Start listener thread
-        expectedResults = len(dataframe) * len(evals)
+        # expectedResults = len(dataframe) * len(evals)
+        expectedResults = len(dataframe)
         # print("expected result" ,expectedResults)
-        timeout = max(100, min(150, expectedResults * 10))
+        timeout = max(100, min(250, expectedResults * 60))
         listener_thread = threading.Thread(
             target=self.socket.listenForResults,
             kwargs={
                 "min_wait": 20,
                 "max_wait": timeout,
-                "inactivity_timeout": 35,
+                "inactivity_timeout": timeout,
                 "expected_results": expectedResults,
             },
             daemon=True,
         )
         listener_thread.start()
-        self.validateApiKey(evalName=evals[0])
+        # commenting validate api key as we don't need it logger does it for us. uncommented but we need different
+        # api for this which don't spend time on eval defintiion fetches and just bring hits
+        self.validateApiKey()
         activePlayground = self.playgroundID
         # print(f"\n======= Running evaluation for: {evalName} =======")
 
         # Validate API and dependencies
         # self.validateApiKey(evalName=evals[0])
-        customAnalytics = getCustomAnalytics(self.workspaceID)
+
+        # why we need custom analytics here? there is no such usage below
+        # customAnalytics = getCustomAnalytics(self.workspaceID)
+
         # metricDependencies = checkDependency(
         #     evalName,
         #     list(dataframe.columns),
@@ -789,37 +876,72 @@ class LlumoClient:
             self.hitsAvailable,
             len(dataframe),
         )
+
+        #where does this remaining hit comes from?
+
+        
         if not userHits["success"]:
             raise LlumoAIError.InsufficientCredits(userHits["message"])
 
         currentBatch = []
+        
+
         for index, row in dataframe.iterrows():
-            tools = [row["tools"]] if "tools" in dataframe.columns else []
+            # Extract required fields
+            tools = row.get("tools", "")
             groundTruth = row.get("groundTruth", "")
-            messageHistory = (
-                [row["messageHistory"]]
-                if "messageHistory" in dataframe.columns
-                else []
-            )
-            promptTemplate = prompt_template
-            keys = re.findall(r"{{(.*?)}}", promptTemplate)
-
-            if not all([ky in dataframe.columns for ky in keys]):
-                raise LlumoAIError.InvalidPromptTemplate()
-
-            inputDict = {key: row[key] for key in keys if key in row}
-            # output = row.get(outputColName, "")
-            output = row.get("output","")
+            messageHistory = row.get("messageHistory", "")
             intermediateSteps = row.get("intermediateSteps", "")
-
+            output = row.get("output", "")
+            
+            # Initialize query and context
+            query = ""
+            context = ""
+            
+            # Process prompt template if provided
+            if prompt_template:
+                # Extract template variables
+                keys = re.findall(r"{{(.*?)}}", prompt_template)
+                
+                if not all([key in dataframe.columns for key in keys]):
+                    raise LlumoAIError.InvalidPromptTemplate()
+                
+                # Populate template and separate query/context
+                populated_template = prompt_template
+                for key in keys:
+                    value = row.get(key, "")
+                    if isinstance(value, str):
+                        length = len(value.split()) * 1.5
+                        if length <= 50:
+                            # Short value - include in query via template
+                            temp_obj = {key: value}
+                            populated_template = getInputPopulatedPrompt(populated_template, temp_obj)
+                        else:
+                            # Long value - add to context
+                            context += f" {key}: {value}, "
+                
+                query = populated_template.strip()
+                
+                # Add any remaining context from other fields
+                if not context.strip():
+                    for key, value in row.items():
+                        if key not in keys and isinstance(value, str) and value.strip():
+                            context += f" {key}: {value}, "
+            else:
+                # No prompt template - use direct query and context fields
+                query = row.get("query", "")
+                context = row.get("context", "")
+            
+            # Generate unique IDs
             rowID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
             columnID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
-
+            
             compoundKey = f"{rowID}-{columnID}-{columnID}"
-            # rowIdMapping[compoundKey] = {"index": index, "eval": evalName}
             rowIdMapping[compoundKey] = {"index": index}
+            print("rowIdMapping:", rowIdMapping)
 
-
+            # Create evaluation payload
+            print("socketID in before templateData: ", socketID)
             templateData = {
                 "processID": getProcessID(),
                 "socketID": socketID,
@@ -832,65 +954,22 @@ class LlumoClient:
                 "playgroundID": activePlayground,
                 "source": "SDK",
                 "processData": {
-                    # "analyticsName": evalName,
-                    # "definition": evalDefinition,
                     "executionDependency": {
-                        "query": "",
-                        "context": "",
+                        "query": query,
+                        "context": context.strip(),
                         "output": output,
                         "tools": tools,
                         "groundTruth": groundTruth,
                         "messageHistory": messageHistory,
                         "intermediateSteps": intermediateSteps,
                     },
-                    "evallist":evals,
-                    # "model": model,
-                    # "provider": provider,
-                    "sessionID":self.sessionID
-                    # "categories": categories,
-                    # "evaluationStrictness": evaluationStrictness,
-                    # "grammarCheckOutput": grammarCheckOutput,
-                    # "insightLength": insightLength,
-                    # "numJudges": numJudges,
-                    # "penaltyBonusInstructions": penaltyBonusInstructions,
-                    # "probableEdgeCases": probableEdgeCases,
+                    "evallist": evals,
+                    "sessionID": self.sessionID
                 },
                 "type": "FULL_EVAL_RUN",
-                # "kpi": evalName,
-                # "fieldMappig": fieldMapping,
             }
 
-            query = ""
-            context = ""
-            for key, value in inputDict.items():
-                if isinstance(value, str):
-                    length = len(value.split()) * 1.5
-                    if length > 50:
-                        context += f" {key}: {value}, "
-                    else:
-                        if promptTemplate:
-                            tempObj = {key: value}
-                            promptTemplate = getInputPopulatedPrompt(
-                                promptTemplate, tempObj
-                            )
-                        else:
-                            query += f" {key}: {value}, "
-
-            if not context.strip():
-                for key, value in inputDict.items():
-                    context += f" {key}: {value}, "
-
-            templateData["processData"]["executionDependency"][
-                "context"
-            ] = context.strip()
-            templateData["processData"]["executionDependency"][
-                "query"
-            ] = query.strip()
-            if promptTemplate and not query.strip():
-                templateData["processData"]["executionDependency"][
-                    "query"
-                ] = promptTemplate
-
+            # Add to batch
             currentBatch.append(templateData)
             if len(currentBatch) == 10:
                 self.allBatches.append(currentBatch)
@@ -920,19 +999,33 @@ class LlumoClient:
 
         rawResults = self.socket.getReceivedData()
 
+        print(f"Total results received: {len(rawResults)}")
+        # print("Raw results:", rawResults)
+
         # print("data from db #####################",dataFromDb)
         # Fix here: keep full keys, do not split keys
         receivedRowIDs = {key for item in rawResults for key in item.keys()}
+        print("Received Row IDs:", receivedRowIDs)
         expectedRowIDs = set(rowIdMapping.keys())
         missingRowIDs = expectedRowIDs - receivedRowIDs
         # print("All expected keys:", expectedRowIDs)
         # print("All received keys:", receivedRowIDs)
         # print("Missing keys:", len(missingRowIDs))
         missingRowIDs = list(missingRowIDs)
-        if len(missingRowIDs) > 0:
-            dataFromDb = fetchData(workspaceID, activePlayground, missingRowIDs)
-            rawResults.extend(dataFromDb)
 
+        print("Missing Row IDs:", missingRowIDs)
+        print(f"Total results before fetching missing data: {len(rawResults)}")
+        if len(missingRowIDs) > 0:
+            print('''It's taking longer than expected to get results for some rows. You can close this now. 
+                  Please wait for 15 mins while we create the flow graph for you. You can check the graph at app.llumo.ai/debugger''')
+        else:
+            print('''All results received successfully. You can check flowgraph in 5 mins at app.llumo.ai/debugger''')
+        # if len(missingRowIDs) > 0:
+        #     dataFromDb = self.fetchDataForMissingKeys(workspaceID, missingRowIDs)
+        #     # print("Fetched missing data from DB:", dataFromDb)
+        #     rawResults.extend(dataFromDb)
+        #     print(f"Total results after fetching missing data: {len(rawResults)}")
+        
         self.evalData = rawResults
         # print("RAW RESULTS: ", self.evalData)
 
@@ -950,9 +1043,6 @@ class LlumoClient:
                 index = rowIdMapping[compound_key]["index"]
                 rowID, columnID, _ = compound_key.split("-", 2)
 
-                if hasattr(self, "startLlumoRun"):
-                    self.startLlumoRun(runName="evaluateMultiple",rowID = rowID, columnID = columnID)
-
                 # get the dataframe row at this index
                 row = dataframe.iloc[index].to_dict()
 
@@ -964,10 +1054,10 @@ class LlumoClient:
                 fullEval = value.get("fullEval") if isinstance(value, dict) else None
                 if fullEval:
                     if "evalMetrics" in fullEval and isinstance(fullEval["evalMetrics"], list):
-                        for eval_item in fullEval["evalMetrics"]:
-                            evalName = eval_item.get("evalName") or eval_item.get("kpiName")
-                            score = str(eval_item.get("score")) or eval_item.get("value")
-                            reasoning = eval_item.get("reasoning")
+                        for evalItem in fullEval["evalMetrics"]:
+                            evalName = evalItem.get("evalName") or evalItem.get("kpiName")
+                            score = str(evalItem.get("score")) or evalItem.get("value")
+                            reasoning = evalItem.get("reasoning")
                             # edgeCase = eval_item.get("edgeCase")
 
                             if evalName:
@@ -975,33 +1065,16 @@ class LlumoClient:
                                 dataframe.at[index, f"{evalName} Reason"] = reasoning
                                 # dataframe.at[index, f"{evalName} EdgeCase"] = edgeCase
 
-                                # logEvalStep if available
-                                if hasattr(self, "logEvalStep"):
-                                    try:
-                                        start_time = time.time()
-                                        self.logEvalStep(
-                                            stepName=f"EVAL-{evalName}",
-                                            output=row.get("output", ""),
-                                            context=row.get("context", ""),
-                                            query=row.get("query", ""),
-                                            messageHistory=row.get("messageHistory", ""),
-                                            tools=row.get("tools", ""),
-                                            intermediateSteps=row.get("intermediateSteps", ""),
-                                            groundTruth=row.get("groundTruth", ""),
-                                            analyticsScore=score,
-                                            reasoning=reasoning,
-                                            classification=eval_item.get("classification", {}),
-                                            evalLabel=eval_item.get("evalLabel", {}),
-                                            latencyMs=int((time.time() - start_time) * 1000),
-                                            status="SUCCESS",
-                                            message="",
-                                        )
-                                    except Exception as e:
-                                        print(f"⚠️ logEvalStep failed: {e}")
-            if hasattr(self, "endLlumoRun"):
-                self.endEvalRun()
-
-        # Clean up and finish
+                                
+                # runLog = value.get("runLog") if isinstance(value, dict) else None
+                # if runLog:
+                #     try:
+                #         self.createRunForEvalMultiple(smartLog=runLog)
+                #     except Exception as e:
+                #         print(f"Error posting smartlog: {e}")
+                        
+
+                
         try:
             self.socket.disconnect()
         except Exception:

{llumo-0.2.32 → llumo-0.2.33}/llumo/helpingFuntions.py

@@ -585,7 +585,7 @@ def checkDependency(selectedEval:list = [], columns:list = [],tocheck=True,_retu
 
 def fetchData(workspaceID, playgroundID, missingList: list):
     # Define the URL and prepare the payload
-    socket_data_url = "https://app.llumo.ai/api/eval/get-awaited"
+    socket_data_url = "http://localhost:4747/api/eval/get-awaited"
     payload = {
         "workspaceID": workspaceID,
         "playgroundID": playgroundID,

{llumo-0.2.32 → llumo-0.2.33}/llumo/sockets.py

@@ -150,7 +150,7 @@ class LlumoSocketClient:
 
     def getReceivedData(self):
         with self._lock:
-            # print("Total received:", len(self._received_data))  # DEBUG
+            print("Total received:", len(self._received_data))  # DEBUG
             return self._received_data.copy()
 
     def disconnect(self):

{llumo-0.2.32 → llumo-0.2.33/llumo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.32
+Version: 0.2.33
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo