llumo 0.2.39__py3-none-any.whl → 0.2.41__py3-none-any.whl

llumo/callback.py

@@ -1,3 +1,4 @@
+from logging import lastResort
 from typing import Any, Dict, List
 from langchain_core.callbacks.base import BaseCallbackHandler
 from langchain_core.messages import BaseMessage
@@ -17,7 +18,7 @@ class LlumoCallbackHandler(BaseCallbackHandler):
             raise ValueError("LlumoSessionContext is required")
 
         self.sessionLogger = session
-        self.sessionLogger.isLangchain = True
+        self.sessionLogger.logger.isLangchain = True
         self.agentType = agentType
 
         # Initialize timing and state variables
@@ -30,6 +31,7 @@ class LlumoCallbackHandler(BaseCallbackHandler):
         # Initialize tracking variables
         self.prompt = ""
         self.searchQuery = ""
+        self.currentInputTokens = 0
         self.currentToolName = None
         self.currentToolInput = None
         self.currentAgentName = None
@@ -48,8 +50,7 @@ class LlumoCallbackHandler(BaseCallbackHandler):
         self.currentAction = ""
         self.currentObservation = ""
         self.isAgentExecution = False
-
-
+        self.availableTools = None
 
     def on_chain_start(self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any) -> None:
         """Called when a chain starts - this includes agent execution"""
@@ -86,8 +87,9 @@ class LlumoCallbackHandler(BaseCallbackHandler):
                 elif "id" in serialized and isinstance(serialized["id"], list):
                     self.currentAgentName = serialized["id"][-1] if serialized["id"] else "unknown"
                 else:
-                    self.currentAgentName = "unknown"
-
+                    self.currentAgentName = kwargs.get("name","unknown")
+            else:
+                self.currentAgentName = kwargs.get("name", "unknown")
             # Check if this is agent execution
             if ("agent" in str(self.currentAgentName).lower() or
                     (serialized and serialized.get("_type") == "agent_executor") or
@@ -124,6 +126,9 @@ class LlumoCallbackHandler(BaseCallbackHandler):
 
     def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None:
         """Called when a chain ends"""
+        # print("ON CHAIN END: ",outputs)
+        # print("ON CHAIN END: ",kwargs)
+
         try:
             if self.isAgentExecution and isinstance(outputs, dict) and "output" in outputs:
                 # Use logAgentStep for final completion
@@ -147,16 +152,27 @@ class LlumoCallbackHandler(BaseCallbackHandler):
 
     def on_llm_start(self, serialized: Dict[str, Any], prompts: List[Any], **kwargs: Any) -> None:
         """Called when LLM starts"""
+        # print("ON LLM START: ",serialized)
+        # print("ON LLM START: ", prompts)
+        # print("ON LLM START: ", kwargs)
+        try:
+            self.availableTools = kwargs["invocation_params"]["functions"]
+        except:
+            self.availableTools = []
+
+
         self.llmStartTime = time.time()
         self.stepTime = time.time()
-
+        # print(prompts)
         if self.prompt == "":
             match = re.search(r"Human:\s*(.*)",prompts[0], re.DOTALL)
+            # allPromptInstructions = " ".join(prompts)
             if match:
                 user_question = match.group(1).strip()
                 self.prompt = user_question # 👉 What is LangChain?
             else:
                 self.prompt = ""
+                # self.allPrompt = allPromptInstructions
 
         # Dynamically get model info
         model = "unknown"
@@ -244,9 +260,11 @@ class LlumoCallbackHandler(BaseCallbackHandler):
         # Ensure we have string values
         output = str(output) if output is not None else ""
         model_name = str(model_name) if model_name is not None else "unknown"
+        self.currentInputTokens = input_tokens
 
         # Parse ReAct reasoning from LLM output if we're in agent execution
         if self.isAgentExecution and output:
+            # print("[AGENT EXECUTOR OUTPUT]")
             self._parse_react_reasoning(output)
         try:
             self.sessionLogger.logLlmStep(
@@ -286,7 +304,7 @@ class LlumoCallbackHandler(BaseCallbackHandler):
                 retrieverSource = str(source),
                 topK = len(documents),
                 chunkSize =  chunkSize,
-                context =  [doc.page_content for doc in documents],
+                context =  " ".join([doc.page_content for doc in documents]),
                 searchQuery =  self.prompt if self.prompt != "" else self.searchQuery,
                 latencyMs = 120,  # mock latency, replace with real timing if needed
                 status = "SUCCESS"
@@ -385,6 +403,9 @@ class LlumoCallbackHandler(BaseCallbackHandler):
         # print(f"[DEBUG] Tool started: {self.currentToolName} with input: {input_str}")
 
     def on_tool_end(self, output: Any, **kwargs: Any) -> None:
+        # print("ON TOOL END: ",output)
+        # print("ON TOOL END: ",kwargs)
+
         """Called when a tool completes execution"""
         duration_ms = int((time.time() - self.toolStartTime) * 1000) if self.toolStartTime else 0
 
@@ -423,7 +444,9 @@ class LlumoCallbackHandler(BaseCallbackHandler):
     def on_agent_action(self, action: AgentAction, **kwargs: Any) -> None:
         """Called when an agent takes an action"""
         self.agentsSteps += 1
+
         # print("ON AGENT ACTION: ", action)
+        # print("ON AGENT ACTION: ", kwargs)
 
         try:
             # Dynamically extract information from action
@@ -452,7 +475,7 @@ class LlumoCallbackHandler(BaseCallbackHandler):
                 agentType=self.agentType,
                 agentName=self.currentAgentName or "unknown",
                 numStepsTaken=self.agentsSteps,
-                tools=[tool_name],
+                tools=self.availableTools,
                 query=self.prompt,
                 status=current_status,
                 # message=f"Executing {tool_name} with input: {tool_input}. Reasoning: {reasoning_text}",
@@ -471,7 +494,7 @@ class LlumoCallbackHandler(BaseCallbackHandler):
 
     def on_agent_error(self, error: Exception, **kwargs: Any) -> None:
         """Called when an agent encounters an error"""
-        # print("ITS A AGENT ERROR:", error)
+        # print("ON AGENT ERROR:", error)
         self.hasErrors = True
         self.lastError = str(error)
 
@@ -491,6 +514,8 @@ class LlumoCallbackHandler(BaseCallbackHandler):
 
     def on_tool_error(self, error: Exception, **kwargs: Any) -> None:
         """Called when a tool encounters an error"""
+        # print("ON TOOL ERROR: ",error)
+        # print("ON TOOL ERROR: ", kwargs)
 
         self.hasErrors = True
         self.lastError = str(error)
@@ -517,6 +542,8 @@ class LlumoCallbackHandler(BaseCallbackHandler):
     def on_chain_error(self, error: Exception, **kwargs: Any) -> None:
         """Called when a chain encounters an error"""
         # print("ITS A CHAIN ERROR:", error)
+        # print("ITS A CHAIN ERROR:", kwargs)
+
         self.hasErrors = True
         self.lastError = str(error)
 
@@ -542,13 +569,13 @@ class LlumoCallbackHandler(BaseCallbackHandler):
                     stepName="Chain Execution Error",
                     model="unknown",
                     provider=self.llmProvider,
-                    inputTokens=0,
+                    inputTokens=self.currentInputTokens,
                     outputTokens=0,
                     # temperature=0.0,
                     # promptTruncated=False,
                     latencyMs=0,
                     prompt=self.prompt,
-                    output="",
+                    output=self.lastError,
                     status="FAILURE",
                     # message=str(error),
                 )

llumo/client.py

@@ -43,9 +43,9 @@ createEvalUrl = "https://backend-api.llumo.ai/api/v1/create-debug-log-for-sdk"
 
 class LlumoClient:
 
-    def __init__(self, api_key, playground_id=None):
+    def __init__(self, api_key, playgroundID=None):
         self.apiKey = api_key
-        self.playgroundID = playground_id
+        self.playgroundID = playgroundID
         self.evalData = []
         self.evals = []
         self.processMapping = {}
@@ -779,7 +779,8 @@ class LlumoClient:
     def debugLogs(
             self,
             data,
-            prompt_template="",
+            promptTemplate="",
+            systemInstructions = ""
 
     ):
         if isinstance(data, dict):
@@ -830,15 +831,15 @@ class LlumoClient:
             context = ""
 
             # Process prompt template if provided
-            if prompt_template:
+            if promptTemplate:
                 # Extract template variables
-                keys = re.findall(r"{{(.*?)}}", prompt_template)
+                keys = re.findall(r"{{(.*?)}}", promptTemplate)
 
                 if not all([key in dataframe.columns for key in keys]):
                     raise LlumoAIError.InvalidPromptTemplate()
 
                 # Populate template and separate query/context
-                populated_template = prompt_template
+                populated_template = promptTemplate
                 for key in keys:
                     value = row.get(key, "")
                     if isinstance(value, str):
@@ -908,7 +909,10 @@ class LlumoClient:
                 "createdAt":createdAt,
                 "columnID":rowID,
                 "rowID":columnID,
-                "latency": random.randint(1000, 1500)
+                "latency": random.randint(1000, 1500),
+                "promptTemplate":promptTemplate,
+                "systemInstructions":systemInstructions
+
             }
 
             allBatches.append(batch)
@@ -918,7 +922,7 @@ class LlumoClient:
 
             try:
                 # print(batch)
-                response = postForListOfSteps(record=batch,workspaceID=workspaceID)
+                response = postForDebugLogs(record=batch,workspaceID=workspaceID)
 
                 # failure case inside response
                 if isinstance(response, dict) and str(response.get("status", "")).lower() == "false":
@@ -2078,6 +2082,115 @@ class LlumoClient:
                     definationMapping=self.definationMapping,
                 )
 
+    def get_evaluate_multiple(
+        self,
+        data,
+        evals
+    ) -> List:
+
+        print("Evaluating multiple data with evals:", data, evals)
+
+        dataID = uuid.uuid4().hex[:36]
+
+        self.validateApiKey()
+
+        if not self.workspaceID:
+            raise LlumoAIError("Workspace ID not found after validation.")
+
+        payload = {
+            "dataID": dataID,
+            "data": data,
+            "evals": evals,
+            "workspaceID": self.workspaceID,
+            "playgroundID": self.playgroundID,
+        }
+
+        print("payload", payload)
+
+        # Create evaluation
+        requests.post(
+            "https://backend-api.llumo.ai/api/v1/sdk/create-evaluation-Multiple",
+            json=payload,
+            headers={
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {self.apiKey}",
+            },
+        )
+
+        final_result_data = []
+
+        cursor = "0-0"
+        limit = 10
+        all_data_fetched = False
+
+        while not all_data_fetched:
+            try:
+                response = requests.get(
+                    "https://backend-api.llumo.ai/api/v1/sdk/poll",
+                    params={
+                        "cursor": cursor,
+                        "dataID": dataID,
+                        "limit": limit,
+                    },
+                )
+
+                response_data = response.json()
+                result_data = response_data.get("debugLog", {})
+                print("resultData", result_data)
+
+                results = result_data.get("results", [])
+                final_result_data.extend(results)
+
+                cursor = result_data.get("nextCursor")
+
+                if len(final_result_data) == len(data):
+                    all_data_fetched = True
+
+                time.sleep(10)
+
+            except Exception as error:
+                print("error", error)
+                all_data_fetched = True
+
+        # Shape results
+        formatted_results = []
+
+        for row in final_result_data:
+            score: Dict[str, float | None] = {}
+            reasoning: Dict[str, str] = {}
+
+            for eval_name in evals:
+                details = row.get(eval_name)
+
+                if isinstance(details, dict):
+                    if isinstance(details.get("value"), (int, float)):
+                        score[eval_name] = details.get("value")
+                    else:
+                        score[eval_name] = details.get("score")
+                    reasoning[eval_name] = details.get("reasoning", "")
+
+                elif "score" in row:
+                    score[eval_name] = (
+                        row["score"] if isinstance(row["score"], (int, float)) else None
+                    )
+                    reasoning[eval_name] = row.get("reasoning", "")
+                else:
+                    score[eval_name] = None
+                    reasoning[eval_name] = ""
+
+            formatted_row = {
+                "context": row.get("context", ""),
+                "query": row.get("query", ""),
+                "output": row.get("output", ""),
+                "score": score,
+                "reasoning": reasoning,
+            }
+
+            print(formatted_row)
+            formatted_results.append(formatted_row)
+
+        return formatted_results
+
 
 class SafeDict(dict):
     def __missing__(self, key):

llumo/helpingFuntions.py

@@ -740,7 +740,7 @@ def getCustomAnalytics(workspaceID):
 
 
 
-def postForListOfSteps(record: {},workspaceID):
+def postForDebugLogs(record: {},workspaceID):
     url = "https://backend-api.llumo.ai/api/v1/get-debug-log-for-upload"
     payload = record
     workspaceID = workspaceID
@@ -763,3 +763,74 @@ def postForListOfSteps(record: {},workspaceID):
 
     except Exception as e:
         return  {"status":"False","exception": str(e)}
+
+
+def removeLLmStep(run: dict) -> dict:
+    """
+    Remove LLM steps that appear immediately before an AGENT step.
+    
+    """
+
+    if not run or "steps" not in run:
+        return run
+
+    steps = run["steps"]
+    indices_to_remove = set()
+    llm_stack = []  # stack of indices where stepType == "LLM"
+
+    for idx, step in enumerate(steps):
+        step_type = step.get("stepType")
+
+        if step_type == "LLM":
+            llm_stack.append(idx)
+
+        elif step_type == "AGENT":
+            if llm_stack:
+                last_llm_idx = llm_stack[-1]
+
+                # ✅ Only remove if LLM is immediately before AGENT
+                if last_llm_idx == idx - 1:
+                    indices_to_remove.add(last_llm_idx)
+                    llm_stack.pop()  # matched, so pop
+
+    # Rebuild steps excluding removed indices
+    cleaned_steps = [
+        step for i, step in enumerate(steps)
+        if i not in indices_to_remove
+    ]
+
+    run["steps"] = cleaned_steps
+    return run
+
+
+def addSelectedTools(run: dict) -> dict:
+    """
+    Populate metadata.toolSelected in AGENT steps based on TOOL executions.
+    """
+
+    if not run or "steps" not in run:
+        return run
+
+    steps = run["steps"]
+    current_agent_step = None
+
+    for step in steps:
+        step_type = step.get("stepType")
+
+        # Track the most recent AGENT step
+        if step_type == "AGENT":
+            current_agent_step = step
+
+            # Ensure toolSelected exists
+            metadata = current_agent_step.get("metadata", {})
+            metadata.setdefault("toolSelected", [])
+            current_agent_step["metadata"] = metadata
+
+        # When TOOL is executed, attach it to last AGENT
+        elif step_type == "TOOL" and current_agent_step:
+            tool_name = step.get("metadata", {}).get("toolName")
+
+            if tool_name:
+                current_agent_step["metadata"]["toolSelected"].append(tool_name)
+
+    return run

llumo/llumoLogger.py

@@ -8,6 +8,7 @@ class LlumoLogger:
         self.workspaceID = None
         self.playgroundID = None
         self.userEmailID = None
+        self.isLangchain = False
         self._authenticate()
 
     def _authenticate(self):

llumo/llumoSessionContext.py

@@ -1,3 +1,4 @@
+
 import contextvars
 import uuid
 from typing import Optional, List, Dict, Any
@@ -6,7 +7,8 @@ import requests
 from .client import LlumoClient
 import math
 import base64
-
+from .helpingFuntions import removeLLmStep
+from .helpingFuntions import addSelectedTools
 import random
 
 _ctxLogger = contextvars.ContextVar("ctxLogger")
@@ -28,14 +30,14 @@ def getLlumoRun():
 
 class LlumoSessionContext(LlumoClient):
     def __init__(self, logger, sessionID: Optional[str] = None):
-        super().__init__(api_key=logger.apiKey, playground_id=logger.getPlaygroundID())
+        super().__init__(api_key=logger.apiKey, playgroundID=logger.getPlaygroundID())
         self.sessionID = sessionID or str(uuid.uuid4().hex[:14])
         self.logger = logger
         self.apiKey = logger.apiKey
         self.threadLogger = None
         self.threadSessionID = None
         self.threadLlumoRun = None
-        self.isLangchain = False
+
 
     def start(self):
         self.threadLogger = _ctxLogger.set(self.logger)
@@ -57,9 +59,9 @@ class LlumoSessionContext(LlumoClient):
         self.end()
 
 
-    def startLlumoRun(self, runName: str, rowID: str = "", columnID: str = "", runID: str = None):
+    def startLlumoRun(self, runName: str,promptTemplate:str = "",systemInstructions:str = "", rowID: str = "", columnID: str = "", runID: str = None):
 
-        if runID == "":
+        if runID == None:
             LlumoRunID = str(uuid.uuid4().hex[:16])
         else:
             LlumoRunID = runID
@@ -82,7 +84,7 @@ class LlumoSessionContext(LlumoClient):
             "sessionID": self.sessionID,
             "playgroundID": self.logger.getPlaygroundID(),
             "workspaceID": self.logger.getWorkspaceID(),
-            "source": "SDK_LANGCHAIN" if self.isLangchain  else "SDK_OTHERS",
+            "source": "SDK_LANGCHAIN" if self.logger.isLangchain  else "SDK_OTHERS",
             "rowID": rowID,
             "columnID": columnID,
             "email": self.logger.getUserEmailID(),
@@ -100,8 +102,9 @@ class LlumoSessionContext(LlumoClient):
                 "totalTokens": "",
                 "cost": "",
                 "modelsUsed": "gpt-4o",
-
-                       }
+                       },
+            "promptTemplate":promptTemplate,
+            "systemInstructions":systemInstructions
         }
 
         self.threadLlumoRun = _ctxLlumoRun.set(llumoRun)
@@ -169,8 +172,15 @@ class LlumoSessionContext(LlumoClient):
 
         try:
             # print("[PAYLOAD]: ",run)
-            payload = run
+
+            payload = removeLLmStep(run)
+            # print("*******PAYLOAD AFTER removeLLmStep*******: ", payload)
+
+            payload = addSelectedTools(payload)
+            # print("********PAYLOAD AFTER addSelectedTools*********: ", payload)
+
             response = requests.post(url, headers=headers, json=payload, timeout=20)
+
             response.raise_for_status()
             # print("[PAYLOAD]: ",response.json())
 
@@ -293,7 +303,7 @@ class LlumoSessionContext(LlumoClient):
             "retrieverSource": retrieverSource,
             "topK": topK,
             "chunkSize":chunkSize,
-            "context": context,
+            "context": " ".join(context),
             "searchQuery": searchQuery,
             "latencyMs": latencyMs,
             "status": status,
@@ -318,6 +328,7 @@ class LlumoSessionContext(LlumoClient):
             "agentName": agentName,
             "numStepsTaken": numStepsTaken,
             "tools": tools,
+            "toolSelected":[],
             "query": query,
             "status": status,
         #     "message": message,

{llumo-0.2.39.dist-info → llumo-0.2.41.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.39
+Version: 0.2.41
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

{llumo-0.2.39.dist-info → llumo-0.2.41.dist-info}/RECORD

@@ -1,20 +1,20 @@
 llumo/__init__.py,sha256=kkuppu7ZPiVZFdnYzJ9BM3syMbYHOSZLpwKwAvGHsnY,311
-llumo/callback.py,sha256=8QkVmOsqg6x60yGc4OzQeVzl50TMRo7NAPTcHKn2uwk,24027
+llumo/callback.py,sha256=XQbImLnd64_B2Iir-FCaJcL5Kmpm3RlXJH2zDRXk_yk,25135
 llumo/callbacks-0.py,sha256=TEIOCWRvk2UYsTmBMBsnlgpqWvr-2y3a6d0w_e96NRM,8958
 llumo/chains.py,sha256=6lCgLseh04RUgc6SahhmvQj82quay2Mi1j8gPUlx8Es,2923
-llumo/client.py,sha256=WKqnOYabThMZN5RXColIWvyf6BY6sFIT5C0s2HT89Xc,79766
+llumo/client.py,sha256=YTNdvtv03BVDNHeLypHQadDN2PpXsyOk0dM8hc7P-Ts,83205
 llumo/exceptions.py,sha256=1OyhN9YL9LcyUPUsqYHq6Rret0udATZAwMVJaio2_Ec,2123
 llumo/execution.py,sha256=nWbJ7AvWuUPcOb6i-JzKRna_PvF-ewZTiK8skS-5n3w,1380
 llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
 llumo/google.py,sha256=6y9YnDFDRHv6-sQNT5LIsV9p31BCN0B9eow5KTRBWfM,2185
-llumo/helpingFuntions.py,sha256=a9mq2e-xV4LeGS-rrzVzf9KGss78b8AKjUhICPc-6yU,28045
-llumo/llumoLogger.py,sha256=7DZR2_QHy0fencng9Nnf9UPmEx8-OZzhvz1QRUp9w6c,2190
-llumo/llumoSessionContext.py,sha256=CvlLQtU8KM-ZpPRWjlyogeauL2vJ3uzvtqZ1CSuMhqI,14400
+llumo/helpingFuntions.py,sha256=eMR2Rq8vw4X5sIESOvjOBrEvyYE00Eq7XlQjV66eVcg,29977
+llumo/llumoLogger.py,sha256=ALM4461jItWcvYL9HzTGmB-X-M77YF_9PTeTPxtmP_E,2223
+llumo/llumoSessionContext.py,sha256=XA2SIXKh62NRz1qxkvhpo7ziOhushG2CkJgjUWlmvFw,14880
 llumo/models.py,sha256=aVEZsOOoQx5LeNtwSyBxqvrINq0izH3QWu_YjsMPE6o,2910
 llumo/openai.py,sha256=VstBzaORe8Tq0feUIIEszzcN1oq6TJfkPviaCr5d3Bw,8950
 llumo/sockets.py,sha256=pfWz1zTEiwqJhdbSy3i3_Y4WlIdJ3cuac11wMePeBS0,6130
-llumo-0.2.39.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
-llumo-0.2.39.dist-info/METADATA,sha256=mRa5uyHhlAXGMOdfjZuGCrcFElf_j7YXSIMfqP0SXvA,1662
-llumo-0.2.39.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-llumo-0.2.39.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
-llumo-0.2.39.dist-info/RECORD,,
+llumo-0.2.41.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
+llumo-0.2.41.dist-info/METADATA,sha256=pamF6aPT4DF9NO1361IN4dcJ63O2YUCxdeMPPut2bd0,1662
+llumo-0.2.41.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+llumo-0.2.41.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
+llumo-0.2.41.dist-info/RECORD,,