llumo 0.2.28__py3-none-any.whl → 0.2.30__py3-none-any.whl

llumo/llumoSessionContext.py

@@ -51,8 +51,14 @@ class LlumoSessionContext(LlumoClient):
     def __exit__(self, excType, excVal, excTb):
         self.end()
 
-    def startLlumoRun(self, runName: str):
+    def startLlumoRun(self, runName: str,rowID= "",columnID = ""):
         LlumoRunID = str(uuid.uuid4().hex[:16])
+
+        # if  rowID =="":
+        #     rowID = str(uuid.uuid4().hex[:16])
+        # if columnID == "":
+        #     columnID = str(uuid.uuid4().hex[:16])
+
         currentTime = datetime(2025, 8, 2, 10, 20, 15, tzinfo=timezone.utc)
         createdAt = currentTime.strftime("%Y-%m-%dT%H:%M:%S.000Z")
         llumoRun = {
@@ -62,8 +68,8 @@ class LlumoSessionContext(LlumoClient):
             "playgroundID": self.logger.getPlaygroundID(),
             "workspaceID": self.logger.getWorkspaceID(),
             "source": "SDK",
-            "rowID": "",
-            "columnID": "",
+            "rowID": rowID,
+            "columnID": columnID,
             "email": self.logger.getUserEmailID(),
             "createdAt": createdAt,
             "createdBy": self.logger.getUserEmailID(),
@@ -96,12 +102,52 @@ class LlumoSessionContext(LlumoClient):
         # STEP 3: Send the payload
         url = "https://app.llumo.ai/api/create-debug-log"
         headers = {
-            "Authorization": f"Bearer {self.apiKey}",
+            "Authorization": f"Bearer {self.logger.getWorkspaceID()}",
             "Content-Type": "application/json",
         }
 
         try:
-            response = requests.post(url, headers=headers, json=run, timeout=10)
+            # print(run)
+            response = requests.post(url, headers=headers, json=run, timeout=20)
+            response.raise_for_status()
+            # print(response.json())
+        except requests.exceptions.Timeout:
+            # print("Request timed out.")
+            pass
+        except requests.exceptions.RequestException as e:
+            pass
+
+        # Cleanup
+        if self.threadLlumoRun:
+            _ctxLlumoRun.reset(self.threadLlumoRun)
+            self.threadLlumoRun = None
+
+    def endEvalRun(self):
+        run = getLlumoRun()
+        if run is None:
+            return
+
+        # STEP 1: Sort steps by timestamp
+        steps = run.get("steps", [])
+        # sorted_steps = sorted(steps, key=lambda s: s.get("timestamp", 0))
+
+        # # STEP 2: Remove timestamp from each step before sending
+        # clean_steps = [
+        #     {k: v for k, v in step.items() if k != "timestamp"} for step in sorted_steps
+        # ]
+        # run["steps"] = clean_steps
+
+        # print(run["runName"])  # optional debug log
+
+        # STEP 3: Send the payload
+        url = "https://backend-api.llumo.ai/api/v1/create-debug-log-for-sdk"
+        headers = {
+            "Authorization": f"Bearer {self.logger.getWorkspaceID()}",
+            "Content-Type": "application/json",
+        }
+        # print(run)
+        try:
+            response = requests.post(url, headers=headers, json={"log":run}, timeout=20)
             response.raise_for_status()
             # print(response.json())
         except requests.exceptions.Timeout:
@@ -145,51 +191,52 @@ class LlumoSessionContext(LlumoClient):
         provider: str,
         inputTokens: int,
         outputTokens: int,
-        temperature: float,
-        promptTruncated: bool,
+        # temperature: float,
+        # promptTruncated: bool,
         latencyMs: int,
-        query: str,
+        prompt: str,
         output: str,
         status: str,
-        message: str,
+        # message: str,
     ):
         metadata = {
             "model": model,
             "provider": provider,
             "inputTokens": inputTokens,
             "outputTokens": outputTokens,
-            "temperature": temperature,
-            "promptTruncated": promptTruncated,
+            # "temperature": temperature,
+            # "promptTruncated": promptTruncated,
             "latencyMs": latencyMs,
-            "query": query,
+            "prompt": prompt,
             "output": output,
             "status": status,
-            "message": message,
+            # "message": message,
         }
+
         self.logStep("LLM", stepName, metadata)
 
     def logRetrieverStep(
         self,
         stepName: str,
         retrieverSource: str,
-        queryVectorType: str,
         topK: int,
-        matchedIDs: List[str],
-        query: str,
+        chunkSize,
+        context : str,
+        searchQuery: str,
         latencyMs: int,
-        status: str,
-        message: str,
+        status: str
     ):
         metadata = {
             "retrieverSource": retrieverSource,
-            "queryVectorType": queryVectorType,
             "topK": topK,
-            "matchedIDs": matchedIDs,
-            "query": query,
+            "chunkSize":chunkSize,
+            "context": context,
+            "searchQuery": searchQuery,
             "latencyMs": latencyMs,
             "status": status,
-            "message": message,
+            # "message": message,
         }
+
         self.logStep("RETRIEVER", stepName, metadata)
 
     def logAgentStep(
@@ -201,7 +248,7 @@ class LlumoSessionContext(LlumoClient):
         tools: List[str],
         query: str,
         status: str,
-        message: str,
+        # message: str,
     ):
         metadata = {
             "agentType": agentType,
@@ -210,8 +257,8 @@ class LlumoSessionContext(LlumoClient):
             "tools": tools,
             "query": query,
             "status": status,
-            "message": message,
-        }
+        #     "message": message,
+         }
         self.logStep("AGENT", stepName, metadata)
 
     def logToolSelectorStep(
@@ -222,7 +269,7 @@ class LlumoSessionContext(LlumoClient):
         selectedTool: str,
         reasoning: str,
         status: str,
-        message: str,
+        # message: str,
     ):
         metadata = {
             "selectorType": selectorType,
@@ -230,7 +277,7 @@ class LlumoSessionContext(LlumoClient):
             "selectedTool": selectedTool,
             "reasoning": reasoning,
             "status": status,
-            "message": message,
+            # "message": message,
         }
         self.logStep("TOOL_SELECTOR", stepName, metadata)
 
@@ -238,19 +285,21 @@ class LlumoSessionContext(LlumoClient):
         self,
         stepName: str,
         toolName: str,
+        description: str,
         input: Dict[str, Any],
         output: str,
         latencyMs: int,
         status: str,
-        message: str,
+        # message: str,
     ):
         metadata = {
             "toolName": toolName,
+            "description":description,
             "input": input,
             "output": output,
             "latencyMs": latencyMs,
             "status": status,
-            "message": message,
+            # "message": message,
         }
         self.logStep("TOOL", stepName, metadata)
 
@@ -364,3 +413,16 @@ class LlumoSessionContext(LlumoClient):
             "message": message,
         }
         self.logStep("CUSTOM_SCRIPT", stepName, metadata)
+
+
+    def logQueryStep(self,stepName,model,provider,inputTokens,query,status):
+        metadata = {
+            "model": model,
+            "provider": provider,
+            "inputTokens": inputTokens,
+            "query": query,
+            "status":status
+        }
+        self.logStep("QUERY", stepName, metadata)
+
+

llumo/openai.py

@@ -12,8 +12,6 @@ def performEvaluation(data, api_key=None, evals=["Response Correctness"], **kwar
         results = client.evaluateMultiple(
             data,
             evals=evals,
-            createExperiment=kwargs.get("createExperiment", False),
-            playgroundID=kwargs.get("playgroundID"),
             prompt_template="Give answer to the query: {{query}}, using context: {{context}}",
             getDataFrame=False,
         )
@@ -25,9 +23,13 @@ def performEvaluation(data, api_key=None, evals=["Response Correctness"], **kwar
 
 # Wrapper around ChatCompletion to allow custom fields like `.evaluation`
 class ChatCompletionWithEval:
-    def __init__(self, response, evaluation):
+    def __init__(
+        self,
+        response,
+        #  , evaluation
+    ):
         self._response = response
-        self.evaluation = evaluation
+        # self.evaluation = evaluation
 
     def __getattr__(self, name):
         return getattr(self._response, name)
@@ -70,70 +72,77 @@ class OpenAI(OpenAIClient):
             workspace_id = self.session.logger.getWorkspaceID()
 
             # Input Bias Evaluation
-            eval_input_bias = [
-                {
-                    "query": user_message,
-                    "context": context,
-                    "output": "",  # No output yet
-                }
-            ]
-            try:
-                start_time = time.time()
-                bias_evaluation_result = performEvaluation(
-                    eval_input_bias,
-                    api_key=self.llumo_key,
-                    evals=["Input Bias"],
-                    playgroundID=playground_id,
-                    workspaceID=workspace_id,
-                    createExperiment=create_experiment,
-                )
-                latency = int((time.time() - start_time) * 1000)
-                # Access the first result object
-                bias_evaluation = bias_evaluation_result[0]
-                message = "-".join(
-                    getattr(bias_evaluation, "edgeCases", {}).get("value", [])
-                )
-                self.session.logEvalStep(
-                    stepName=f"EVAL-Input Bias",
-                    output="",
-                    context=context,
-                    query=user_message,
-                    messageHistory="",
-                    tools="",
-                    intermediateSteps="",
-                    groundTruth="",
-                    analyticsScore=getattr(bias_evaluation, "analyticsScore", {}),
-                    reasoning=getattr(bias_evaluation, "reasoning", {}),
-                    classification=getattr(bias_evaluation, "classification", {}),
-                    evalLabel=getattr(bias_evaluation, "evalLabel", {}),
-                    latencyMs=latency,
-                    status="SUCCESS",
-                    message=message,
-                )
-            except Exception as e:
-                print(f"Input Bias evaluation failed: {e}")
-                self.session.logEvalStep(
-                    stepName=f"EVAL-FAILURE",
-                    output="",
-                    context=context,
-                    query=user_message,
-                    messageHistory="",
-                    tools="",
-                    intermediateSteps="",
-                    groundTruth="",
-                    analyticsScore={},
-                    reasoning={},
-                    classification={},
-                    evalLabel={},
-                    latencyMs=0,
-                    status="FAILURE",
-                    message="EVAL_ERROR",
-                )
+            # eval_input_bias = [
+            #     {
+            #         "query": user_message,
+            #         "context": context,
+            #         "output": "",  # No output yet
+            #     }
+            # ]
+            # try:
+            #     start_time = time.time()
+            #     bias_evaluation_result = performEvaluation(
+            #         eval_input_bias,
+            #         api_key=self.llumo_key,
+            #         evals=["Input Bias"],
+            #         playgroundID=playground_id,
+            #         workspaceID=workspace_id,
+            #         createExperiment=create_experiment,
+            #     )
+            #     latency = int((time.time() - start_time) * 1000)
+            #     # Access the first result object
+            #     bias_evaluation = bias_evaluation_result[0]
+            #     message = "-".join(
+            #         getattr(bias_evaluation, "edgeCases", {}).get("value", [])
+            #     )
+            #     self.session.logEvalStep(
+            #         stepName=f"EVAL-Input Bias",
+            #         output="",
+            #         context=context,
+            #         query=user_message,
+            #         messageHistory="",
+            #         tools="",
+            #         intermediateSteps="",
+            #         groundTruth="",
+            #         analyticsScore=getattr(bias_evaluation, "analyticsScore", {}),
+            #         reasoning=getattr(bias_evaluation, "reasoning", {}),
+            #         classification=getattr(bias_evaluation, "classification", {}),
+            #         evalLabel=getattr(bias_evaluation, "evalLabel", {}),
+            #         latencyMs=latency,
+            #         status="SUCCESS",
+            #         message=message,
+            #     )
+            # except Exception as e:
+            #     print(f"Input Bias evaluation failed: {e}")
+            #     self.session.logEvalStep(
+            #         stepName=f"EVAL-FAILURE",
+            #         output="",
+            #         context=context,
+            #         query=user_message,
+            #         messageHistory="",
+            #         tools="",
+            #         intermediateSteps="",
+            #         groundTruth="",
+            #         analyticsScore={},
+            #         reasoning={},
+            #         classification={},
+            #         evalLabel={},
+            #         latencyMs=0,
+            #         status="FAILURE",
+            #         message="EVAL_ERROR",
+            #     )
 
             start_time = time.time()
             response = original_create(*args, **kwargs)
             latency = int((time.time() - start_time) * 1000)
             output_text = response.choices[0].message.content
+            self.session.logQueryStep(
+                stepName="Query Invocation",
+                model=model,
+                provider="openai",
+                inputTokens=response.usage.prompt_tokens,
+                query=user_message,
+                status = "SUCCESS")
 
             self.session.logLlmStep(
                 stepName=f"LLM-{user_message[:30]}",
@@ -141,84 +150,87 @@ class OpenAI(OpenAIClient):
                 provider="openai",
                 inputTokens=response.usage.prompt_tokens,
                 outputTokens=response.usage.completion_tokens,
-                temperature=kwargs.get("temperature", 0.0),
-                promptTruncated=False,
+                # temperature=kwargs.get("temperature", 0.0),
+                # promptTruncated=False,
                 latencyMs=latency,
-                query=user_message,
+                prompt=user_message,
                 output=output_text,
                 status="SUCCESS",
-                message="",
+                # message="",
             )
 
             # Response Correctness Evaluation
-            eval_input_correctness = [
-                {
-                    "query": user_message,
-                    "context": context,
-                    "output": output_text,
-                }
-            ]
-            try:
-                start_time = time.time()
-                correctness_evaluation_result = performEvaluation(
-                    eval_input_correctness,
-                    api_key=self.llumo_key,
-                    evals=["Response Correctness"],
-                    playgroundID=playground_id,
-                    workspaceID=workspace_id,
-                    createExperiment=create_experiment,
-                )
-                latency = int((time.time() - start_time) * 1000)
-                # Access the first result object
-                correctness_evaluation = correctness_evaluation_result[0]
-                message = "-".join(
-                    getattr(correctness_evaluation, "edgeCases", {}).get("value", [])
-                )
-                self.session.logEvalStep(
-                    stepName=f"EVAL-Response Correctness",
-                    output=output_text,
-                    context=context,
-                    query=user_message,
-                    messageHistory="",
-                    tools="",
-                    intermediateSteps="",
-                    groundTruth="",
-                    analyticsScore=getattr(
-                        correctness_evaluation, "analyticsScore", {}
-                    ),
-                    reasoning=getattr(correctness_evaluation, "reasoning", {}),
-                    classification=getattr(
-                        correctness_evaluation, "classification", {}
-                    ),
-                    evalLabel=getattr(correctness_evaluation, "evalLabel", {}),
-                    latencyMs=latency,
-                    status="SUCCESS",
-                    message=message,
-                )
-            except Exception as e:
-                print(f"Response Correctness evaluation failed: {e}")
-                correctness_evaluation = None
-                self.session.logEvalStep(
-                    stepName=f"EVAL-FAILURE",
-                    output=output_text,
-                    context=context,
-                    query=user_message,
-                    messageHistory="",
-                    tools="",
-                    intermediateSteps="",
-                    groundTruth="",
-                    analyticsScore={},
-                    reasoning={},
-                    classification={},
-                    evalLabel={},
-                    latencyMs=0,
-                    status="FAILURE",
-                    message="EVAL_ERROR",
-                )
-
-            if correctness_evaluation is None:
-                return response
-
-            return ChatCompletionWithEval(response, correctness_evaluation)
+            # eval_input_correctness = [
+            #     {
+            #         "query": user_message,
+            #         "context": context,
+            #         "output": output_text,
+            #     }
+            # ]
+            # try:
+            #     start_time = time.time()
+            #     correctness_evaluation_result = performEvaluation(
+            #         eval_input_correctness,
+            #         api_key=self.llumo_key,
+            #         evals=["Response Correctness"],
+            #         playgroundID=playground_id,
+            #         workspaceID=workspace_id,
+            #         createExperiment=create_experiment,
+            #     )
+            #     latency = int((time.time() - start_time) * 1000)
+            #     # Access the first result object
+            #     correctness_evaluation = correctness_evaluation_result[0]
+            #     message = "-".join(
+            #         getattr(correctness_evaluation, "edgeCases", {}).get("value", [])
+            #     )
+            #     self.session.logEvalStep(
+            #         stepName=f"EVAL-Response Correctness",
+            #         output=output_text,
+            #         context=context,
+            #         query=user_message,
+            #         messageHistory="",
+            #         tools="",
+            #         intermediateSteps="",
+            #         groundTruth="",
+            #         analyticsScore=getattr(
+            #             correctness_evaluation, "analyticsScore", {}
+            #         ),
+            #         reasoning=getattr(correctness_evaluation, "reasoning", {}),
+            #         classification=getattr(
+            #             correctness_evaluation, "classification", {}
+            #         ),
+            #         evalLabel=getattr(correctness_evaluation, "evalLabel", {}),
+            #         latencyMs=latency,
+            #         status="SUCCESS",
+            #         message=message,
+            #     )
+            # except Exception as e:
+            #     print(f"Response Correctness evaluation failed: {e}")
+            #     correctness_evaluation = None
+            #     self.session.logEvalStep(
+            #         stepName=f"EVAL-FAILURE",
+            #         output=output_text,
+            #         context=context,
+            #         query=user_message,
+            #         messageHistory="",
+            #         tools="",
+            #         intermediateSteps="",
+            #         groundTruth="",
+            #         analyticsScore={},
+            #         reasoning={},
+            #         classification={},
+            #         evalLabel={},
+            #         latencyMs=0,
+            #         status="FAILURE",
+            #         message="EVAL_ERROR",
+            #     )
+
+            # if correctness_evaluation is None:
+            #     return response
+
+            return ChatCompletionWithEval(
+                response
+                # , correctness_evaluation
+            )
 
         self.chat.completions.create = create_wrapper

{llumo-0.2.28.dist-info → llumo-0.2.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.28
+Version: 0.2.30
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

llumo-0.2.30.dist-info/RECORD

@@ -0,0 +1,20 @@
+llumo/__init__.py,sha256=kkuppu7ZPiVZFdnYzJ9BM3syMbYHOSZLpwKwAvGHsnY,311
+llumo/callback.py,sha256=K8O_bXgeIOXr5gougWF2Y7wnXTf7c3gSEvJZFiVoCmA,23829
+llumo/callbacks-0.py,sha256=TEIOCWRvk2UYsTmBMBsnlgpqWvr-2y3a6d0w_e96NRM,8958
+llumo/chains.py,sha256=6lCgLseh04RUgc6SahhmvQj82quay2Mi1j8gPUlx8Es,2923
+llumo/client.py,sha256=_icK1unxsPdKiLboLlRcrpvfO6IuaZeoS-n3j5eZIX8,70772
+llumo/exceptions.py,sha256=1OyhN9YL9LcyUPUsqYHq6Rret0udATZAwMVJaio2_Ec,2123
+llumo/execution.py,sha256=nWbJ7AvWuUPcOb6i-JzKRna_PvF-ewZTiK8skS-5n3w,1380
+llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
+llumo/google.py,sha256=6y9YnDFDRHv6-sQNT5LIsV9p31BCN0B9eow5KTRBWfM,2185
+llumo/helpingFuntions.py,sha256=RSXMA5XLP6qiMsJW7axp1PZj8RFwY9HkkbG7nwJkbIM,27279
+llumo/llumoLogger.py,sha256=grdjhu6Ngxg7nhnrMOP5Pd5ALR7U2ROws48yhf_N7y0,1912
+llumo/llumoSessionContext.py,sha256=eBlVRDeRmJ8BzfrHNofEiy8DO_TZCwzLo06LlOBcyho,12417
+llumo/models.py,sha256=aVEZsOOoQx5LeNtwSyBxqvrINq0izH3QWu_YjsMPE6o,2910
+llumo/openai.py,sha256=VstBzaORe8Tq0feUIIEszzcN1oq6TJfkPviaCr5d3Bw,8950
+llumo/sockets.py,sha256=pBDo-U65hMIMwKMwZQl3iBkEjISEt-9BkXxZTWfSHF4,6116
+llumo-0.2.30.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
+llumo-0.2.30.dist-info/METADATA,sha256=VstCu1Qq9l_1qrerrHmTt1KFTByH62V1RK2c1XzB78g,1558
+llumo-0.2.30.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+llumo-0.2.30.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
+llumo-0.2.30.dist-info/RECORD,,

llumo-0.2.28.dist-info/RECORD

@@ -1,20 +0,0 @@
-llumo/__init__.py,sha256=kkuppu7ZPiVZFdnYzJ9BM3syMbYHOSZLpwKwAvGHsnY,311
-llumo/callback.py,sha256=dOsQ35Ro6IVec3TiJfkPx3H9PQtk8oWfJA1skFENTIM,20439
-llumo/callbacks-0.py,sha256=TEIOCWRvk2UYsTmBMBsnlgpqWvr-2y3a6d0w_e96NRM,8958
-llumo/chains.py,sha256=6lCgLseh04RUgc6SahhmvQj82quay2Mi1j8gPUlx8Es,2923
-llumo/client.py,sha256=keYx0GToNB-FXmGncXd0MOwwCGjxsIoDbOhTx2rCcMQ,71582
-llumo/exceptions.py,sha256=1OyhN9YL9LcyUPUsqYHq6Rret0udATZAwMVJaio2_Ec,2123
-llumo/execution.py,sha256=nWbJ7AvWuUPcOb6i-JzKRna_PvF-ewZTiK8skS-5n3w,1380
-llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
-llumo/google.py,sha256=3S_aRtbtlctCXPGR0u4baLlkyFrsjd02vlUCkoRPA5U,2147
-llumo/helpingFuntions.py,sha256=B6FwUQ5f1v4FKrWCbYoGWMFdscOV_liuuhTgNQ3cdrk,27275
-llumo/llumoLogger.py,sha256=grdjhu6Ngxg7nhnrMOP5Pd5ALR7U2ROws48yhf_N7y0,1912
-llumo/llumoSessionContext.py,sha256=iJDeC3FG6dID9jdaDK5d6hIlXWuhWx4KQy-Nnty4wYg,10485
-llumo/models.py,sha256=aVEZsOOoQx5LeNtwSyBxqvrINq0izH3QWu_YjsMPE6o,2910
-llumo/openai.py,sha256=c0pZ-yzm6LfUAbfVmOiVpY9pS5sAWZRb8_jAj0ir910,8450
-llumo/sockets.py,sha256=pBDo-U65hMIMwKMwZQl3iBkEjISEt-9BkXxZTWfSHF4,6116
-llumo-0.2.28.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
-llumo-0.2.28.dist-info/METADATA,sha256=WYFBghjzGVN4xRdMjqsg3QSKTw5np1qzBoy5tIbcTvk,1558
-llumo-0.2.28.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-llumo-0.2.28.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
-llumo-0.2.28.dist-info/RECORD,,