llumo 0.2.42__tar.gz → 0.2.44__tar.gz

{llumo-0.2.42/llumo.egg-info → llumo-0.2.44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.42
+Version: 0.2.44
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

{llumo-0.2.42 → llumo-0.2.44}/llumo/client.py

@@ -8,10 +8,12 @@ import json
 import uuid
 import warnings
 import os
-
+from typing import List, Dict, Optional
 import itertools
 import pandas as pd
-from typing import List, Dict
+from typing import List, Dict, Any
+
+
 from .models import AVAILABLEMODELS, getProviderFromModel, Provider
 from .execution import ModelExecutor
 from .exceptions import LlumoAIError
@@ -50,6 +52,7 @@ class LlumoClient:
         self.evals = []
         self.processMapping = {}
         self.definationMapping = {}
+        self.ALL_USER_AIM =  ['incorrectOutput', 'incorrectInput', 'hallucination', 'ragQuality', 'contextMismanagement', 'toolCallIssues', 'agentReasoning', 'stuckAgents', 'jsonErrors', 'highLatency', 'highCost', 'safetyBlocks', 'modelRouting', 'systemErrors', 'promptAdherence']
 
     def validateApiKey(self, evalName="Input Bias"):
         headers = {
@@ -780,7 +783,9 @@ class LlumoClient:
             self,
             data,
             promptTemplate="",
-            systemInstructions = ""
+            systemInstructions = "",
+            multiTurnChat=False,
+            createMultipleLogs = True
 
     ):
         if isinstance(data, dict):
@@ -788,7 +793,7 @@ class LlumoClient:
         elif not isinstance(data, list):
             raise ValueError("Data should be a dict or a list of dicts.")
 
-        dataframe = pd.DataFrame(data).astype(str)
+        dataframe = pd.DataFrame(data).astype(str).replace(to_replace="nan",value = "")
         workspaceID = None
         email = None
 
@@ -822,25 +827,45 @@ class LlumoClient:
             # Extract required fields
             query = row.get("query", "")
             context = row.get("context", "")
+
             tools = row.get("tools", "")
             groundTruth = row.get("groundTruth", "")
-            messageHistory = row.get("messageHistory", "")
+
+            if  multiTurnChat==False:
+                # ---- SINGLE TURN (existing behavior) ----
+                messageHistory = row.get("messageHistory", "")
+
+            else:
+                # ---- MULTI TURN ----
+                multiTurnData = createMessageHistory(data, index)
+
+                if createMultipleLogs==True:
+                    # each row will get history till that point
+                    messageHistory = multiTurnData
+                else:
+                    # only final API call should contain full history
+                    if index == len(dataframe) - 1:
+                        messageHistory = multiTurnData
+                    else:
+                        messageHistory = ""
+
+
+
             intermediateSteps = row.get("intermediateSteps", "")
             output = row.get("output", "")
 
-            # Initialize query and context
+            # # Initialize query and context
             # query = ""
             # context = ""
-
+            #
             # # Process prompt template if provided
             # if promptTemplate:
             #     # Extract template variables
             #     keys = re.findall(r"{{(.*?)}}", promptTemplate)
-
+            #
             #     if not all([key in dataframe.columns for key in keys]):
-            #         # raise LlumoAIError.InvalidPromptTemplate()
-            #         break 
-                    
+            #         raise LlumoAIError.InvalidPromptTemplate()
+            #
             #     # Populate template and separate query/context
             #     populated_template = promptTemplate
             #     for key in keys:
@@ -854,9 +879,9 @@ class LlumoClient:
             #             else:
             #                 # Long value - add to context
             #                 context += f" {key}: {value}, "
-
+            #
             #     query = populated_template.strip()
-
+            #
             #     # Add any remaining context from other fields
             #     if not context.strip():
             #         for key, value in row.items():
@@ -894,6 +919,7 @@ class LlumoClient:
 
 
             }
+
             currentTime = datetime(2025, 8, 2, 10, 20, 15, tzinfo=timezone.utc)
             createdAt = currentTime.strftime("%Y-%m-%dT%H:%M:%S.000Z")
             rowID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
@@ -926,7 +952,7 @@ class LlumoClient:
         for i, batch in enumerate(allBatches, start=1):
 
             try:
-                #  print(batch)
+                # print(batch)
                 response = postForDebugLogs(record=batch,workspaceID=workspaceID)
 
                 # failure case inside response
@@ -943,6 +969,9 @@ class LlumoClient:
 
         print("Records Uploaded successfully. You may now review your logs at: https://app.llumo.ai/logs")
 
+
+        # Wait for results
+
     # def evaluateMultiple(
     #     self,
     #     data,
@@ -2084,14 +2113,149 @@ class LlumoClient:
                     definationMapping=self.definationMapping,
                 )
 
+    # def get_evaluate_multiple(
+    #     self,
+    #     data,
+    #     evals
+    # ) -> List:
+
+    #     print("Evaluating multiple data with evals:", data, evals)
+
+    #     dataID = uuid.uuid4().hex[:36]
+
+    #     self.validateApiKey()
+
+    #     if not self.workspaceID:
+    #         raise LlumoAIError("Workspace ID not found after validation.")
+
+    #     payload = {
+    #         "dataID": dataID,
+    #         "data": data,
+    #         "evals": evals,
+    #         "workspaceID": self.workspaceID,
+    #         "playgroundID": self.playgroundID,
+    #     }
+
+    #     print("payload", payload)
+
+    #     # Create evaluation
+    #     requests.post(
+    #         "https://backend-api.llumo.ai/api/v1/sdk/create-evaluation-Multiple",
+    #         json=payload,
+    #         headers={
+    #             "Content-Type": "application/json",
+    #             "Authorization": f"Bearer {self.apiKey}",
+    #         },
+    #     )
+
+    #     final_result_data = []
+
+    #     cursor = "0-0"
+    #     limit = 10
+    #     all_data_fetched = False
+
+    #     while not all_data_fetched:
+    #         try:
+    #             response = requests.get(
+    #                 "https://backend-api.llumo.ai/api/v1/sdk/poll",
+    #                 params={
+    #                     "cursor": cursor,
+    #                     "dataID": dataID,
+    #                     "limit": limit,
+    #                 },
+    #             )
+
+    #             response_data = response.json()
+    #             result_data = response_data.get("debugLog", {})
+    #             print("resultData", result_data)
+
+    #             results = result_data.get("results", [])
+    #             final_result_data.extend(results)
+
+    #             cursor = result_data.get("nextCursor")
+
+    #             if len(final_result_data) == len(data):
+    #                 all_data_fetched = True
+
+    #             time.sleep(10)
+
+    #         except Exception as error:
+    #             print("error", error)
+    #             all_data_fetched = True
+
+    #     # Shape results
+    #     formatted_results = []
+
+    #     for row in final_result_data:
+    #         score: Dict[str, float | None] = {}
+    #         reasoning: Dict[str, str] = {}
+
+    #         for eval_name in evals:
+    #             details = row.get(eval_name)
+
+    #             if isinstance(details, dict):
+    #                 if isinstance(details.get("value"), (int, float)):
+    #                     score[eval_name] = details.get("value")
+    #                 else:
+    #                     score[eval_name] = details.get("score")
+    #                 reasoning[eval_name] = details.get("reasoning", "")
+
+    #             elif "score" in row:
+    #                 score[eval_name] = (
+    #                     row["score"] if isinstance(row["score"], (int, float)) else None
+    #                 )
+    #                 reasoning[eval_name] = row.get("reasoning", "")
+    #             else:
+    #                 score[eval_name] = None
+    #                 reasoning[eval_name] = ""
+
+    #         formatted_row = {
+    #             "context": row.get("context", ""),
+    #             "query": row.get("query", ""),
+    #             "output": row.get("output", ""),
+    #             "score": score,
+    #             "reasoning": reasoning,
+    #         }
+
+    #         print(formatted_row)
+    #         formatted_results.append(formatted_row)
+
+    #     return formatted_results
+
     def getEvaluateMultiple(
-        self,
-        data,
-        evals
-    ) -> List:
+            self,
+            data,
+            evals,
+            promptTemplate="",
+            systemInstructions="",
+            multiTurnChat=False,
+            createMultipleLogs=True
+    ):
 
         # print("Evaluating multiple data with evals:", data, evals)
+        rawData = data.copy()
+        try:
+            self.validateApiKey()
+        except Exception as e:
+            print(f"Error during API key validation: {str(e)}")
+            if hasattr(e, "response") and getattr(e, "response", None) is not None:
+                print(f"Status code: {e.response.status_code}")
+                print(f"Response content: {e.response.text[:500]}...")
+            raise
 
+        userHits = checkUserHits(
+            self.workspaceID,
+            self.hasSubscribed,
+            self.trialEndDate,
+            self.subscriptionEndDate,
+            self.hitsAvailable,
+            len(data),
+        )
+
+        if not userHits["success"]:
+            raise LlumoAIError.InsufficientCredits(userHits["message"])
+
+        print("✅ SDK integration successful!")
         dataID = uuid.uuid4().hex[:36]
 
         self.validateApiKey()
@@ -2099,6 +2263,38 @@ class LlumoClient:
         if not self.workspaceID:
             raise LlumoAIError("Workspace ID not found after validation.")
 
+        if promptTemplate:
+            for row in data:
+                row["promptTemplate"] = promptTemplate
+        if systemInstructions:
+            for row in data:
+                row["systemInstructions"] = systemInstructions
+
+        if multiTurnChat == True:
+            if createMultipleLogs == True:
+                dataWithMessageHistory = []
+                for indx, row in enumerate(data):
+                    messageHistory = createMessageHistory(data, currentIndex=indx)
+                    rowCopy = row.copy()
+                    rowCopy["messageHistory"] = messageHistory
+                    dataWithMessageHistory.append(rowCopy)
+                data = dataWithMessageHistory
+            else:
+                dataWithMessageHistory = []
+                for indx, row in enumerate(data):
+                    if indx == len(data) - 1:
+                        messageHistory = createMessageHistory(data, currentIndex=indx)
+                        rowCopy = row.copy()
+                        rowCopy["messageHistory"] = messageHistory
+                        dataWithMessageHistory.append(rowCopy)
+                    else:
+                        row["messageHistory"] = ""
+                        dataWithMessageHistory.append(row)
+
+                data = dataWithMessageHistory
+
+        # print("DATA:")
+        # print(data)
         payload = {
             "dataID": dataID,
             "data": data,
@@ -2107,91 +2303,128 @@ class LlumoClient:
             "playgroundID": self.playgroundID,
         }
 
-        #  print("payload", payload)
+        # print("payload", payload)
 
-        # Create evaluation
-        requests.post(
-            "https://backend-api.llumo.ai/api/v1/sdk/create-evaluation-Multiple",
-            json=payload,
-            headers={
-                "Content-Type": "application/json",
-                "Authorization": f"Bearer {self.apiKey}",
-            },
+        # Create evaluation + Poll results (moved to helper)
+        final_result_data = dataPollingFuncForEval(
+            api_key=self.apiKey,
+            payload=payload,
+            data=data,
         )
+        # Shape results
+        formatted_results = []
 
-        final_result_data = []
 
-        cursor = "0-0"
-        limit = 10
-        all_data_fetched = False
+        for row in final_result_data:
+            # print("ROW:  ",row)
+            result = []
 
-        while not all_data_fetched:
-            try:
-                response = requests.get(
-                    "https://backend-api.llumo.ai/api/v1/sdk/poll",
-                    params={
-                        "cursor": cursor,
-                        "dataID": dataID,
-                        "limit": limit,
-                    },
-                )
+            # Extract numeric keys ("0", "1", "2", ...)
+            numeric_keys = sorted(
+                [key for key in row.keys() if str(key).strip() != "" and str(key).isdigit()],
+                key=lambda x: int(x)
+            )
 
-                response_data = response.json()
-                result_data = response_data.get("debugLog", {})
-                print("resultData", result_data)
+            for key in numeric_keys:
+                result.append(row[key])
 
-                results = result_data.get("results", [])
-                final_result_data.extend(results)
+            evalData={}
+            for key in row:
+                if key not in numeric_keys:
+                    evalData[key]=row[key]
 
-                cursor = result_data.get("nextCursor")
+            # evalResultDict = {"evaluation": result}
+            evalData = {}
+            for key in row:
+                if key not in numeric_keys:
+                    evalData[key] = row[key]
 
-                if len(final_result_data) == len(data):
-                    all_data_fetched = True
+            # evalResultDict = {"evaluation": result}
 
-                time.sleep(10)
+            evalData["evaluation"] = result
+            formatted_results.append(evalData)
 
-            except Exception as error:
-                print("error", error)
-                all_data_fetched = True
+        return {"llumoEval": formatted_results}
+        # return formatted_results
 
-        # Shape results
-        formatted_results = []
+    def getInsights(self,logs:List,userAim:List[str],promptTemplate:str = ""
+                    ,systemInstructions:str="",multiTurnChat=False,createMultipleLogs=True):
 
-        for row in final_result_data:
-            score: Dict[str, float | None] = {}
-            reasoning: Dict[str, str] = {}
+        try:
+            self.validateApiKey()
+        except Exception as e:
+            print(f"Error during API key validation: {str(e)}")
+            if hasattr(e, "response") and getattr(e, "response", None) is not None:
+                print(f"Status code: {e.response.status_code}")
+                print(f"Response content: {e.response.text[:500]}...")
+            raise
+
+        userHits = checkUserHits(
+            self.workspaceID,
+            self.hasSubscribed,
+            self.trialEndDate,
+            self.subscriptionEndDate,
+            self.hitsAvailable,
+            1,
+        )
+
+        if not userHits["success"]:
+            raise LlumoAIError.InsufficientCredits(userHits["message"])
 
-            for eval_name in evals:
-                details = row.get(eval_name)
+        if len(logs)==0 :
+            raise LlumoAIError.emptyLogList()
 
-                if isinstance(details, dict):
-                    if isinstance(details.get("value"), (int, float)):
-                        score[eval_name] = details.get("value")
+        if not isinstance(userAim, list):
+            raise TypeError(f"userAim must be list, got {type(userAim).__name__}")
+
+        if any(aim not in self.ALL_USER_AIM for aim in userAim):
+            errorMessage = f"Please pass a valid user aim. Only acceptable user aims are->{self.ALL_USER_AIM}"
+            raise LlumoAIError.invalidUserAim(details=errorMessage)
+
+
+
+        if multiTurnChat == True:
+            if createMultipleLogs == True:
+                dataWithMessageHistory = []
+                for indx, row in enumerate(logs):
+                    messageHistory = createMessageHistory(logs, currentIndex=indx)
+                    rowCopy = row.copy()
+                    rowCopy["messageHistory"] = messageHistory
+                    dataWithMessageHistory.append(rowCopy)
+                logs= dataWithMessageHistory
+            else:
+                dataWithMessageHistory = []
+                for indx, row in enumerate(logs):
+                    if indx == len(logs) - 1:
+                        messageHistory = createMessageHistory(logs, currentIndex=indx)
+                        rowCopy = row.copy()
+                        rowCopy["messageHistory"] = messageHistory
+                        dataWithMessageHistory.append(rowCopy)
                     else:
-                        score[eval_name] = details.get("score")
-                    reasoning[eval_name] = details.get("reasoning", "")
+                        row["messageHistory"] = ""
+                        dataWithMessageHistory.append(row)
+                logs = dataWithMessageHistory
 
-                elif "score" in row:
-                    score[eval_name] = (
-                        row["score"] if isinstance(row["score"], (int, float)) else None
-                    )
-                    reasoning[eval_name] = row.get("reasoning", "")
-                else:
-                    score[eval_name] = None
-                    reasoning[eval_name] = ""
-
-            formatted_row = {
-                "context": row.get("context", ""),
-                "query": row.get("query", ""),
-                "output": row.get("output", ""),
-                "score": score,
-                "reasoning": reasoning,
-            }
+        if (promptTemplate!="") or systemInstructions!="":
+            logs = addPromptAndInstructionInLogs(logData = logs ,promptTemplate=promptTemplate,systemInstruction=systemInstructions)
+
+        # print("[LOGS: ]")
+        # print(logs)
+
+        # 1. Create Report
+        print("✅ Generating Insights Now....")
+        dataID = uuid.uuid4().hex[:36]
+        payload = {
+            "data":logs,
+            "userAim":userAim,
+            "dataID":dataID
+        }
 
-            # print(formatted_row)
-            formatted_results.append(formatted_row)
+        # 2. Poll for Results
+        insight_result = dataPollingFuncForInsight(payload)
+        # llumoInsight = formattedInsightResponse(llmResponse=insight_result)
 
-        return formatted_results
+        return {"llumoInsight": insight_result}
 
 
 class SafeDict(dict):

{llumo-0.2.42 → llumo-0.2.44}/llumo/exceptions.py

@@ -1,3 +1,4 @@
+
 class LlumoAIError(Exception):
     """Base class for all Llumo SDK-related errors."""
 
@@ -54,6 +55,14 @@ class LlumoAIError(Exception):
     def providerError(details):
         return LlumoAIError(details)
 
+    @staticmethod
+    def emptyLogList(details= "List of log object is empty. Ensure your logs have at least 1 log object."):
+        return LlumoAIError(details)
+
+    @staticmethod
+    def invalidUserAim(details= ""):
+        return LlumoAIError(details)
+
     # @staticmethod
     # def dateNotFound():
     #     return LlumoAIError("Trial end date or subscription end date not found for the given user.")

{llumo-0.2.42 → llumo-0.2.44}/llumo/helpingFuntions.py

@@ -29,6 +29,9 @@ createInsightUrl="https://app.llumo.ai/api/external/generate-insight-from-eval-f
 
 getCustomAnalyticsUrl="https://app.llumo.ai/api/workspace/get-all-analytics"
 
+
+
+
 def getProcessID():
     return f"{int(time.time() * 1000)}{uuid.uuid4()}"
 
@@ -739,6 +742,8 @@ def getCustomAnalytics(workspaceID):
         return {}
 
 
+def normalize_md(s: str) -> str:
+    return "\n".join(line.lstrip() for line in s.splitlines())
 
 def postForDebugLogs(record: {},workspaceID):
     url = "https://backend-api.llumo.ai/api/v1/get-debug-log-for-upload"
@@ -834,3 +839,167 @@ def addSelectedTools(run: dict) -> dict:
                 current_agent_step["metadata"]["toolSelected"].append(tool_name)
 
     return run
+
+def createMessageHistory(data, currentIndex = 0):
+    conversationHistory = []
+    for dataObj in data[:currentIndex]:
+        conversationHistory.append(dataObj)
+    return f"{conversationHistory}"
+
+
+def addPromptAndInstructionInLogs(logData=None,promptTemplate= "",systemInstruction=""):
+    logDataWithPrompt = []
+    for data in logData:
+        if isinstance(data,str):
+            logDataWithPrompt.append(data + f"**promptTemplate**={promptTemplate}\n **systemInstruction**={systemInstruction}")
+        elif isinstance(data,dict):
+            data["promptTemplate"]= promptTemplate
+            data["systemInstruction"]= systemInstruction
+            logDataWithPrompt.append(data)
+
+        else:
+            return logData
+    return logDataWithPrompt
+
+
+def dataPollingFuncForEval(api_key, payload, data, poll_interval=10, limit=10):
+    # Create evaluation (POST)
+    requests.post(
+        "https://backend-api.llumo.ai/api/v1/sdk/create-evaluation-Multiple",
+        json=payload,
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}",
+        },
+    )
+
+    final_result_data = []
+
+    cursor = "0-0"
+    all_data_fetched = False
+
+    print("✅ Evaluation Started...")
+    while not all_data_fetched:
+        try:
+            response = requests.get(
+                "https://backend-api.llumo.ai/api/v1/sdk/poll",
+                params={
+                    "cursor": cursor,
+                    "dataID": payload["dataID"],
+                    "limit": limit,
+                },
+            )
+
+            response_data = response.json()
+            result_data = response_data.get("debugLog", {})
+
+            results = result_data.get("results", [])
+            final_result_data.extend(results)
+
+            cursor = result_data.get("nextCursor")
+
+            if len(final_result_data) == len(data):
+                all_data_fetched = True
+
+            time.sleep(poll_interval)
+
+        except Exception as error:
+            print("error", error)
+            all_data_fetched = True
+
+    return final_result_data
+
+def dataPollingFuncForInsight(payload, poll_interval=1, limit=10, max_polls=20):
+    dataID = payload["dataID"]
+
+    # -------------------------------
+    # 1. Create Insight Report (POST)
+    # -------------------------------
+    try:
+        create_url = "https://backend-api.llumo.ai/api/v1/sdk/create-insight-report"
+        response = requests.post(create_url, json=payload)
+        response.raise_for_status()
+    except requests.exceptions.RequestException as e:
+        error_data = e.response.json() if e.response else str(e)
+        print(f"Error in create request: {error_data}")
+        return None
+
+    # -------------------------------
+    # 2. Poll Insight Results (GET)
+    # -------------------------------
+    cursor = "0-0"
+    poll_count = 0
+    insight_result = []
+
+    while poll_count < max_polls:
+        poll_count += 1
+
+        try:
+            poll_params = {
+                "dataID": dataID,
+                "cursor": cursor,
+                "limit": limit,
+            }
+
+            poll_url = "https://backend-api.llumo.ai/api/v1/sdk/poll-insight-report"
+            poll_response = requests.get(poll_url, params=poll_params)
+            poll_response.raise_for_status()
+
+            data = poll_response.json()
+            debug_log = data.get("debugLog", {})
+            results = debug_log.get("results", [])
+            next_cursor = debug_log.get("nextCursor")
+
+            if results:
+                insight_result.extend(results)
+                break  # same heuristic as original
+
+            if next_cursor != cursor:
+                cursor = next_cursor
+
+        except requests.exceptions.RequestException as e:
+            if e.response is not None and e.response.status_code == 404:
+                pass
+            else:
+                break
+
+        time.sleep(poll_interval)
+
+    return insight_result
+
+def formattedInsightResponse(llmResponse):
+    try:
+        response = llmResponse
+        jsonResponse = response.json().get("response", {})
+        reasons = "\n- ".join(jsonResponse['reason'])
+        solutions = "\n- ".join(jsonResponse['solution'])
+        examples = "\n- ".join(jsonResponse['examples'])
+
+        formattedResponse = f"""
+        # 🔍 **{jsonResponse['insightTitle'].strip()}**
+        ---
+        ## 🧠 **Insight**
+        > {jsonResponse['insight'].strip()}
+        ---
+        ## 📝 **Description**
+        {jsonResponse['shortDescription'].strip()}
+        ---
+        ## 🔎 **Root Causes**
+        - {reasons.strip()}
+        ---
+        ## 🛠 **Solutions**
+        - {solutions.strip()}
+        ---
+        ## 📌 **Examples**
+        - {examples.strip()}
+        ---
+        """
+        formattedResponse = normalize_md(formattedResponse)
+        llumoInsight = formattedResponse
+
+
+
+    except Exception as e:
+        print("An error occurred. Please try again.")
+        llumoInsight = e
+    return llumoInsight

{llumo-0.2.42 → llumo-0.2.44}/llumo/llumoLogger.py

@@ -2,9 +2,10 @@ import requests
 
 
 class LlumoLogger:
-    def __init__(self, apiKey: str, playground: str):
+    def __init__(self, apiKey: str, project: str):
         self.apiKey = apiKey
-        self.playground = playground
+        # self.playground = playground
+        self.playground = project
         self.workspaceID = None
         self.playgroundID = None
         self.userEmailID = None

{llumo-0.2.42 → llumo-0.2.44/llumo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.42
+Version: 0.2.44
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo