llumo 0.2.43__tar.gz → 0.2.45__tar.gz

{llumo-0.2.43/llumo.egg-info → llumo-0.2.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.43
+Version: 0.2.45
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

{llumo-0.2.43 → llumo-0.2.45}/llumo/client.py

@@ -2231,7 +2231,9 @@ class LlumoClient:
             multiTurnChat=False,
             createMultipleLogs=True
     ):
+
         # print("Evaluating multiple data with evals:", data, evals)
+        rawData = data.copy()
         try:
             self.validateApiKey()
         except Exception as e:
@@ -2303,91 +2305,47 @@ class LlumoClient:
 
         # print("payload", payload)
 
-        # Create evaluation
-        requests.post(
-            "https://backend-api.llumo.ai/api/v1/sdk/create-evaluation-Multiple",
-            json=payload,
-            headers={
-                "Content-Type": "application/json",
-                "Authorization": f"Bearer {self.apiKey}",
-            },
+        # Create evaluation + Poll results (moved to helper)
+        final_result_data = dataPollingFuncForEval(
+            api_key=self.apiKey,
+            payload=payload,
+            data=data,
         )
-
-        final_result_data = []
-
-        cursor = "0-0"
-        limit = 10
-        all_data_fetched = False
-
-        print("✅ Evaluation Started...")
-        while not all_data_fetched:
-            try:
-                response = requests.get(
-                    "https://backend-api.llumo.ai/api/v1/sdk/poll",
-                    params={
-                        "cursor": cursor,
-                        "dataID": dataID,
-                        "limit": limit,
-                    },
-                )
-
-                response_data = response.json()
-                result_data = response_data.get("debugLog", {})
-                # print("resultData", result_data)
-
-                results = result_data.get("results", [])
-                final_result_data.extend(results)
-
-                cursor = result_data.get("nextCursor")
-
-                if len(final_result_data) == len(data):
-                    all_data_fetched = True
-
-                time.sleep(10)
-
-            except Exception as error:
-                print("error", error)
-                all_data_fetched = True
-
         # Shape results
         formatted_results = []
 
+
         for row in final_result_data:
-            score: Dict[str, float | None] = {}
-            reasoning: Dict[str, str] = {}
+            # print("ROW:  ",row)
+            result = []
 
-            for eval_name in evals:
-                details = row.get(eval_name)
+            # Extract numeric keys ("0", "1", "2", ...)
+            numeric_keys = sorted(
+                [key for key in row.keys() if str(key).strip() != "" and str(key).isdigit()],
+                key=lambda x: int(x)
+            )
 
-                if isinstance(details, dict):
-                    if isinstance(details.get("value"), (int, float)):
-                        score[eval_name] = details.get("value")
-                    else:
-                        score[eval_name] = details.get("score")
-                    reasoning[eval_name] = details.get("reasoning", "")
+            for key in numeric_keys:
+                result.append(row[key])
 
-                elif "score" in row:
-                    score[eval_name] = (
-                        row["score"] if isinstance(row["score"], (int, float)) else None
-                    )
-                    reasoning[eval_name] = row.get("reasoning", "")
-                else:
-                    score[eval_name] = None
-                    reasoning[eval_name] = ""
-
-            formatted_row = {
-                "query": row.get("query", ""),
-                "context": row.get("context", ""),
-                "output": row.get("output", ""),
-                "score": score,
-                "reasoning": reasoning,
-            }
+            evalData={}
+            for key in row:
+                if key not in numeric_keys:
+                    evalData[key]=row[key]
 
-            # print(formatted_row)
-            formatted_results.append(formatted_row)
+            # evalResultDict = {"evaluation": result}
+            evalData = {}
+            for key in row:
+                if key not in numeric_keys:
+                    evalData[key] = row[key]
 
-        return {"llumoEval": formatted_results}
+            # evalResultDict = {"evaluation": result}
 
+            evalData["evaluation"] = result
+            formatted_results.append(evalData)
+
+        return {"llumoEval": formatted_results}
+        # return formatted_results
 
     def getInsights(self,logs:List,userAim:List[str],promptTemplate:str = ""
                     ,systemInstructions:str="",multiTurnChat=False,createMultipleLogs=True):
@@ -2456,80 +2414,14 @@ class LlumoClient:
         # 1. Create Report
         print("✅ Generating Insights Now....")
         dataID = uuid.uuid4().hex[:36]
-        try:
-            payload = {
-                "data":logs,
-                "userAim":userAim,
-                "dataID":dataID
-            }
-            create_url = "https://backend-api.llumo.ai/api/v1/sdk/create-insight-report"
-            response = requests.post(create_url, json=payload)
-
-            # Check if request was successful
-            response.raise_for_status()
-
-            # print(f"Create Response: {response.json()}")
-        except requests.exceptions.RequestException as e:
-            # Check for response data in the error
-            error_data = e.response.json() if e.response else str(e)
-            print(f"Error in create request: {error_data}")
-            return
+        payload = {
+            "data":logs,
+            "userAim":userAim,
+            "dataID":dataID
+        }
 
         # 2. Poll for Results
-        # print("\n--- Starting Polling ---")
-        cursor = '0-0'
-        is_complete = False
-        max_polls = 20
-        poll_count = 0
-
-        insight_result = []
-        while not is_complete and poll_count < max_polls:
-            poll_count += 1
-
-            try:
-                poll_params = {
-                    'dataID': dataID,
-                    'cursor': cursor,
-                    'limit': 10
-                }
-                poll_url = f"https://backend-api.llumo.ai/api/v1/sdk/poll-insight-report"
-                poll_response = requests.get(poll_url, params=poll_params)
-                poll_response.raise_for_status()
-
-                data = poll_response.json()
-                # Accessing nested data: pollResponse.data.debugLog
-                debug_log = data.get('debugLog', {})
-                results = debug_log.get('results', [])
-                next_cursor = debug_log.get('nextCursor')
-
-                if results:
-                    insight_result.extend(results)
-
-                # Logic to handle cursor movement
-                if next_cursor == cursor and not results:
-                    # print("Long poll returned empty (timeout). Continuing...")
-                    pass
-                else:
-                    cursor = next_cursor
-
-                # Break condition for test (heuristic)
-                if len(insight_result):
-                    break
-
-            except requests.exceptions.RequestException as e:
-                error_msg = e.response.json() if e.response else str(e)
-                # print(f"Error in poll request: {error_msg}")
-
-                if e.response is not None and e.response.status_code == 404:
-                    # print("Resource not ready yet...")
-                    pass
-                else:
-                    # Fatal error, break the loop
-                    break
-
-            # Small delay to prevent tight loop (Equivalent to await sleep(1000))
-            time.sleep(1)
-
+        insight_result = dataPollingFuncForInsight(payload)
         # llumoInsight = formattedInsightResponse(llmResponse=insight_result)
 
         return {"llumoInsight": insight_result}

llumo-0.2.45/llumo/exceptions.py

@@ -0,0 +1,105 @@
+
+class LlumoAIError(Exception):
+    """Base class for all Llumo SDK-related errors."""
+
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.message)
+
+    @staticmethod
+    def InvalidApiKey():
+        return LlumoAIError("The provided API key is invalid or unauthorized"
+                            "To fix this:\n"
+        "1. Go to https://app.llumo.ai/getting-started\n"
+        "2. Look at the top navigation bar (right side)\n"
+        "3. Copy the API key shown under “API Key”\n"
+        "4. Paste that key into your SDK configuration"
+)
+
+    @staticmethod
+    def InvalidApiResponse():
+        return LlumoAIError("Invalid or UnexpectedError response from the API"
+                            "We received a response from the API, but it wasn’t in the expected format…”")
+
+    @staticmethod
+    def RequestFailed(detail="The request could not be completed."):
+        return LlumoAIError(
+            f"We were unable to complete the request to the Llumo API. "
+            f"{detail} "
+            "Please check your network connection or try again later."
+        )
+
+
+    @staticmethod
+    def InvalidJsonResponse():
+        return LlumoAIError("The API response is not in valid JSON format")
+
+    @staticmethod
+    def UnexpectedError(detail="Metric"):
+        return LlumoAIError(
+            f"We couldn’t find an evaluation named '{detail}'. "
+            f"Please check that the name is correct. "
+            f"If you’d like to run '{detail}', you can create a custom evaluation "
+            f"with the same name at https://app.llumo.ai/evallm."
+        )
+
+    @staticmethod
+    def EvalError(detail="Some error occured while processing"):
+        return LlumoAIError(f"error: {detail}")
+
+    @staticmethod
+    def InsufficientCredits(details="Your available credits have been exhausted."):
+        return LlumoAIError(
+            f"{details} "
+            "To continue running evaluations, please upgrade your plan or "
+            "increase your usage limits in the LLUMO AI dashboard at: "
+            "https://app.llumo.ai/settings."
+        )
+
+        # return LlumoAIError("LLumo hits exhausted")
+
+    @staticmethod
+    def InvalidPromptTemplate():
+        return LlumoAIError('''Make sure the prompt template fulfills the following criteria:
+        1. All the variables should be inside double curly braces. Example: Give answer for the {{query}}, based on given {{context}}.
+        2. The variables used in the prompt template must be present in the dataframe columns with the same name..
+        ''')
+
+    @staticmethod
+    def modelHitsExhausted(details = "Your credits for the selected model exhausted."):
+        return LlumoAIError(details)
+
+    @staticmethod
+    def dependencyError(details):
+        return LlumoAIError(details)
+
+    @staticmethod
+    def providerError(details):
+        return LlumoAIError(details)
+
+    @staticmethod
+    def emptyLogList(details="No logs were provided for analysis."):
+        return LlumoAIError(
+            f"{details} "
+            "Please pass at least one log entry. "
+            "You can find the correct log format at "
+            "https://app.llumo.ai/getting-started "
+            "under the “Run SDK with zero data egress” section."
+        )
+
+    @staticmethod
+    def invalidUserAim(details=""):
+        return LlumoAIError(
+            "Invalid userAim detected. "
+            "Each userAim must match one of the supported categories used for analysis. "
+            "Valid options include:\n"
+            "[incorrectOutput, incorrectInput, hallucination, ragQuality, "
+            "contextMismanagement, toolCallIssues, agentReasoning, stuckAgents, "
+            "jsonErrors, highLatency, highCost, safetyBlocks, modelRouting, "
+            "systemErrors, promptAdherence]."
+        )
+
+
+    # @staticmethod
+    # def dateNotFound():
+    #     return LlumoAIError("Trial end date or subscription end date not found for the given user.")

{llumo-0.2.43 → llumo-0.2.45}/llumo/helpingFuntions.py

@@ -862,48 +862,111 @@ def addPromptAndInstructionInLogs(logData=None,promptTemplate= "",systemInstruct
     return logDataWithPrompt
 
 
-def dataPollingFuncForEval(headers,post_payload,poll_params,data_len:int):
-    postUrl = "https://backend-api.llumo.ai/api/v1/sdk/create-evaluation-Multiple"
-    pollUrl = "https://backend-api.llumo.ai/api/v1/sdk/poll"
-
-    # Create evaluation
+def dataPollingFuncForEval(api_key, payload, data, poll_interval=10, limit=10):
+    # Create evaluation (POST)
     requests.post(
-        url= postUrl,
-        json=post_payload,
-        headers=headers,
+        "https://backend-api.llumo.ai/api/v1/sdk/create-evaluation-Multiple",
+        json=payload,
+        headers={
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}",
+        },
     )
-    # print("payload", payload)
 
     final_result_data = []
+
+    cursor = "0-0"
     all_data_fetched = False
 
     print("✅ Evaluation Started...")
     while not all_data_fetched:
         try:
             response = requests.get(
-                pollUrl,
-                params=poll_params,
+                "https://backend-api.llumo.ai/api/v1/sdk/poll",
+                params={
+                    "cursor": cursor,
+                    "dataID": payload["dataID"],
+                    "limit": limit,
+                },
             )
 
             response_data = response.json()
             result_data = response_data.get("debugLog", {})
-            # print("resultData", result_data)
 
             results = result_data.get("results", [])
             final_result_data.extend(results)
 
             cursor = result_data.get("nextCursor")
 
-            if len(final_result_data) == data_len:
+            if len(final_result_data) == len(data):
                 all_data_fetched = True
 
-            time.sleep(10)
+            time.sleep(poll_interval)
 
         except Exception as error:
             print("error", error)
             all_data_fetched = True
+
     return final_result_data
 
+def dataPollingFuncForInsight(payload, poll_interval=1, limit=10, max_polls=20):
+    dataID = payload["dataID"]
+
+    # -------------------------------
+    # 1. Create Insight Report (POST)
+    # -------------------------------
+    try:
+        create_url = "https://backend-api.llumo.ai/api/v1/sdk/create-insight-report"
+        response = requests.post(create_url, json=payload)
+        response.raise_for_status()
+    except requests.exceptions.RequestException as e:
+        error_data = e.response.json() if e.response else str(e)
+        print(f"Error in create request: {error_data}")
+        return None
+
+    # -------------------------------
+    # 2. Poll Insight Results (GET)
+    # -------------------------------
+    cursor = "0-0"
+    poll_count = 0
+    insight_result = []
+
+    while poll_count < max_polls:
+        poll_count += 1
+
+        try:
+            poll_params = {
+                "dataID": dataID,
+                "cursor": cursor,
+                "limit": limit,
+            }
+
+            poll_url = "https://backend-api.llumo.ai/api/v1/sdk/poll-insight-report"
+            poll_response = requests.get(poll_url, params=poll_params)
+            poll_response.raise_for_status()
+
+            data = poll_response.json()
+            debug_log = data.get("debugLog", {})
+            results = debug_log.get("results", [])
+            next_cursor = debug_log.get("nextCursor")
+
+            if results:
+                insight_result.extend(results)
+                break  # same heuristic as original
+
+            if next_cursor != cursor:
+                cursor = next_cursor
+
+        except requests.exceptions.RequestException as e:
+            if e.response is not None and e.response.status_code == 404:
+                pass
+            else:
+                break
+
+        time.sleep(poll_interval)
+
+    return insight_result
+
 def formattedInsightResponse(llmResponse):
     try:
         response = llmResponse

{llumo-0.2.43 → llumo-0.2.45}/llumo/llumoSessionContext.py

@@ -11,6 +11,7 @@ from .helpingFuntions import removeLLmStep
 from .helpingFuntions import addSelectedTools
 import random
 
+
 _ctxLogger = contextvars.ContextVar("ctxLogger")
 _ctxSessionID = contextvars.ContextVar("ctxSessionID")
 _ctxLlumoRun = contextvars.ContextVar("ctxLlumoRun")
@@ -179,7 +180,7 @@ class LlumoSessionContext(LlumoClient):
 
             payload = addSelectedTools(payload)
             # print("********PAYLOAD AFTER addSelectedTools*********: ", payload)
-
+            
             response = requests.post(url, headers=headers, json=payload, timeout=20)
 
             response.raise_for_status()

{llumo-0.2.43 → llumo-0.2.45/llumo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.43
+Version: 0.2.45
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

llumo-0.2.43/llumo/exceptions.py

@@ -1,68 +0,0 @@
-
-class LlumoAIError(Exception):
-    """Base class for all Llumo SDK-related errors."""
-
-    def __init__(self, message):
-        self.message = message
-        super().__init__(self.message)
-
-    @staticmethod
-    def InvalidApiKey():
-        return LlumoAIError("The provided API key is invalid or unauthorized")
-
-    @staticmethod
-    def InvalidApiResponse():
-        return LlumoAIError("Invalid or UnexpectedError response from the API")
-
-    @staticmethod
-    def RequestFailed(detail="The request to the API failed"):
-        return LlumoAIError(f"Request to the API failed: {detail}")
-
-    @staticmethod
-    def InvalidJsonResponse():
-        return LlumoAIError("The API response is not in valid JSON format")
-
-    @staticmethod
-    def UnexpectedError(detail="Metric"):
-        return LlumoAIError(f"Can you please check if {detail} is written correctly. If you want to run {detail} please create a custom eval with same name of app.llumo.ai/evallm ")
-
-    @staticmethod
-    def EvalError(detail="Some error occured while processing"):
-        return LlumoAIError(f"error: {detail}")
-
-    @staticmethod
-    def InsufficientCredits(details):
-        return LlumoAIError(details)
-
-        # return LlumoAIError("LLumo hits exhausted")
-
-    @staticmethod
-    def InvalidPromptTemplate():
-        return LlumoAIError('''Make sure the prompt template fulfills the following criteria:
-        1. All the variables should be inside double curly braces. Example: Give answer for the {{query}}, based on given {{context}}.
-        2. The variables used in the prompt template must be present in the dataframe columns with the same name..
-        ''')
-
-    @staticmethod
-    def modelHitsExhausted(details = "Your credits for the selected model exhausted."):
-        return LlumoAIError(details)
-
-    @staticmethod
-    def dependencyError(details):
-        return LlumoAIError(details)
-
-    @staticmethod
-    def providerError(details):
-        return LlumoAIError(details)
-
-    @staticmethod
-    def emptyLogList(details= "List of log object is empty. Ensure your logs have at least 1 log object."):
-        return LlumoAIError(details)
-
-    @staticmethod
-    def invalidUserAim(details= ""):
-        return LlumoAIError(details)
-
-    # @staticmethod
-    # def dateNotFound():
-    #     return LlumoAIError("Trial end date or subscription end date not found for the given user.")