llumo 0.2.24__py3-none-any.whl → 0.2.26__py3-none-any.whl

llumo/llumoLogger.py

@@ -0,0 +1,57 @@
+import requests
+
+
+class LlumoLogger:
+    def __init__(self, apiKey: str, playground: str):
+        self.apiKey = apiKey
+        self.playground = playground
+        self.workspaceID = None
+        self.playgroundID = None
+        self.userEmailID = None
+        self._authenticate()
+
+    def _authenticate(self):
+        url = "https://app.llumo.ai/api/get-playground-name"
+        try:
+            response = requests.post(
+                url,
+                headers={
+                    "Authorization": f"Bearer {self.apiKey}",
+                    "Content-Type": "application/json",
+                },
+                json={"playgroundName": self.playground},
+                timeout=10,
+            )
+
+            response.raise_for_status()
+            res_json = response.json()
+
+            # Navigate into the nested "data" structure
+            inner_data = res_json.get("data", {}).get("data", {})
+
+            self.workspaceID = inner_data.get("workspaceID")
+            self.playgroundID = inner_data.get("playgroundID")
+            self.userEmailID = inner_data.get("createdBy")
+
+            if not self.workspaceID or not self.playgroundID:
+                raise RuntimeError(
+                    f"Invalid response: workspaceID or playgroundID missing. Full response: {res_json}"
+                )
+
+        except requests.exceptions.RequestException as req_err:
+            raise RuntimeError(
+                f"Network or HTTP error during authentication: {req_err}"
+            )
+        except ValueError as json_err:
+            raise RuntimeError(f"Invalid JSON in authentication response: {json_err}")
+        except Exception as e:
+            raise RuntimeError(f"Authentication failed: {e}")
+
+    def getWorkspaceID(self):
+        return self.workspaceID
+
+    def getUserEmailID(self):
+        return self.userEmailID
+
+    def getPlaygroundID(self):
+        return self.playgroundID

llumo/llumoSessionContext.py

@@ -0,0 +1,366 @@
+import contextvars
+import uuid
+from typing import Optional, List, Dict, Any
+from datetime import datetime, timezone
+import requests
+from .client import LlumoClient
+
+_ctxLogger = contextvars.ContextVar("ctxLogger")
+_ctxSessionID = contextvars.ContextVar("ctxSessionID")
+_ctxLlumoRun = contextvars.ContextVar("ctxLlumoRun")
+
+
+def getLogger():
+    return _ctxLogger.get()
+
+
+def getSessionID():
+    return _ctxSessionID.get()
+
+
+def getLlumoRun():
+    return _ctxLlumoRun.get()
+
+
+class LlumoSessionContext(LlumoClient):
+    def __init__(self, logger, sessionID: Optional[str] = None):
+        super().__init__(api_key=logger.apiKey, playground_id=logger.getPlaygroundID())
+        self.sessionID = sessionID or str(uuid.uuid4().hex[:14])
+        self.logger = logger
+        self.apiKey = logger.apiKey
+        self.threadLogger = None
+        self.threadSessionID = None
+        self.threadLlumoRun = None
+
+    def start(self):
+        self.threadLogger = _ctxLogger.set(self.logger)
+        self.threadSessionID = _ctxSessionID.set(self.sessionID)
+
+    def end(self):
+        if self.threadLogger:
+            _ctxLogger.reset(self.threadLogger)
+        if self.threadSessionID:
+            _ctxSessionID.reset(self.threadSessionID)
+        if self.threadLlumoRun:
+            _ctxLlumoRun.reset(self.threadLlumoRun)
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, excType, excVal, excTb):
+        self.end()
+
+    def startLlumoRun(self, runName: str):
+        LlumoRunID = str(uuid.uuid4().hex[:16])
+        currentTime = datetime(2025, 8, 2, 10, 20, 15, tzinfo=timezone.utc)
+        createdAt = currentTime.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+        llumoRun = {
+            "logID": LlumoRunID,
+            "runName": runName,
+            "sessionID": self.sessionID,
+            "playgroundID": self.logger.getPlaygroundID(),
+            "workspaceID": self.logger.getWorkspaceID(),
+            "source": "SDK",
+            "rowID": "",
+            "columnID": "",
+            "email": self.logger.getUserEmailID(),
+            "createdAt": createdAt,
+            "createdBy": self.logger.getUserEmailID(),
+            "status": "SUCCESS",
+            "flow": [],
+            "latency": 4200,
+            "feedback": "",
+            "dump": "",
+            "steps": [],
+        }
+        self.threadLlumoRun = _ctxLlumoRun.set(llumoRun)
+
+    def endLlumoRun(self):
+        run = getLlumoRun()
+        if run is None:
+            return
+
+        # STEP 1: Sort steps by timestamp
+        steps = run.get("steps", [])
+        sorted_steps = sorted(steps, key=lambda s: s.get("timestamp", 0))
+
+        # STEP 2: Remove timestamp from each step before sending
+        clean_steps = [
+            {k: v for k, v in step.items() if k != "timestamp"} for step in sorted_steps
+        ]
+        run["steps"] = clean_steps
+
+        print(run["runName"])  # optional debug log
+
+        # STEP 3: Send the payload
+        url = "https://app.llumo.ai/api/create-debug-log"
+        headers = {
+            "Authorization": f"Bearer {self.apiKey}",
+            "Content-Type": "application/json",
+        }
+
+        try:
+            response = requests.post(url, headers=headers, json=run, timeout=10)
+            response.raise_for_status()
+            # print(response.json())
+        except requests.exceptions.Timeout:
+            print("Request timed out.")
+        except requests.exceptions.RequestException as e:
+            print(f"Request failed: {e}")
+
+        # Cleanup
+        if self.threadLlumoRun:
+            _ctxLlumoRun.reset(self.threadLlumoRun)
+            self.threadLlumoRun = None
+
+    def logStep(
+        self,
+        stepType: str,
+        stepName: str,
+        metadata: Optional[dict] = None,
+    ):
+        print(f"logged: {stepType}")
+        run = getLlumoRun()
+        if run is None:
+            raise RuntimeError("No active run to log steps.")
+
+        # add step
+        stepData = {
+            "stepType": stepType,
+            "stepName": stepName,
+            "status": metadata.get("status", "SUCCESS"),
+            "message": metadata.get("message", ""),
+            "metadata": metadata or {},
+            "timestamp": datetime.now(timezone.utc).timestamp(),  # OPTIONAL
+        }
+        run["steps"].append(stepData)
+        # set to context vars again in llumo run
+        self.threadLlumoRun = _ctxLlumoRun.set(run)
+
+    def logLlmStep(
+        self,
+        stepName: str,
+        model: str,
+        provider: str,
+        inputTokens: int,
+        outputTokens: int,
+        temperature: float,
+        promptTruncated: bool,
+        latencyMs: int,
+        query: str,
+        output: str,
+        status: str,
+        message: str,
+    ):
+        metadata = {
+            "model": model,
+            "provider": provider,
+            "inputTokens": inputTokens,
+            "outputTokens": outputTokens,
+            "temperature": temperature,
+            "promptTruncated": promptTruncated,
+            "latencyMs": latencyMs,
+            "query": query,
+            "output": output,
+            "status": status,
+            "message": message,
+        }
+        self.logStep("LLM", stepName, metadata)
+
+    def logRetrieverStep(
+        self,
+        stepName: str,
+        retrieverSource: str,
+        queryVectorType: str,
+        topK: int,
+        matchedIDs: List[str],
+        query: str,
+        latencyMs: int,
+        status: str,
+        message: str,
+    ):
+        metadata = {
+            "retrieverSource": retrieverSource,
+            "queryVectorType": queryVectorType,
+            "topK": topK,
+            "matchedIDs": matchedIDs,
+            "query": query,
+            "latencyMs": latencyMs,
+            "status": status,
+            "message": message,
+        }
+        self.logStep("RETRIEVER", stepName, metadata)
+
+    def logAgentStep(
+        self,
+        stepName: str,
+        agentType: str,
+        agentName: str,
+        numStepsTaken: int,
+        tools: List[str],
+        query: str,
+        status: str,
+        message: str,
+    ):
+        metadata = {
+            "agentType": agentType,
+            "agentName": agentName,
+            "numStepsTaken": numStepsTaken,
+            "tools": tools,
+            "query": query,
+            "status": status,
+            "message": message,
+        }
+        self.logStep("AGENT", stepName, metadata)
+
+    def logToolSelectorStep(
+        self,
+        stepName: str,
+        selectorType: str,
+        toolsRanked: List[Dict[str, Any]],
+        selectedTool: str,
+        reasoning: str,
+        status: str,
+        message: str,
+    ):
+        metadata = {
+            "selectorType": selectorType,
+            "toolsRanked": toolsRanked,
+            "selectedTool": selectedTool,
+            "reasoning": reasoning,
+            "status": status,
+            "message": message,
+        }
+        self.logStep("TOOL_SELECTOR", stepName, metadata)
+
+    def logToolStep(
+        self,
+        stepName: str,
+        toolName: str,
+        input: Dict[str, Any],
+        output: str,
+        latencyMs: int,
+        status: str,
+        message: str,
+    ):
+        metadata = {
+            "toolName": toolName,
+            "input": input,
+            "output": output,
+            "latencyMs": latencyMs,
+            "status": status,
+            "message": message,
+        }
+        self.logStep("TOOL", stepName, metadata)
+
+    def logEvalStep(
+        self,
+        stepName: str,
+        output: str,
+        context: str,
+        query: str,
+        # total 7 keys add 4 more
+        messageHistory: str,
+        tools: str,
+        intermediateSteps: str,
+        groundTruth: str,
+        analyticsScore: Dict[str, float],
+        reasoning: Dict[str, str],
+        classification: Dict[str, str],
+        evalLabel: Dict[str, str],
+        latencyMs: int,
+        status: str,
+        message: str,
+    ):
+        metadata = {
+            "output": output,
+            "context": context,
+            "query": query,
+            "messageHistory": messageHistory,
+            "tools": tools,
+            "intermediateSteps": intermediateSteps,
+            "groundTruth": groundTruth,
+            "analyticsScore": analyticsScore,
+            "reasoning": reasoning,
+            "classification": classification,
+            "evalLabel": evalLabel,
+            "latencyMs": latencyMs,
+            "status": status,
+            "message": message,
+        }
+        self.logStep("EVAL", stepName, metadata)
+
+    def logFunctionCallStep(
+        self,
+        stepName: str,
+        functionName: str,
+        argsPassed: Dict[str, Any],
+        output: Dict[str, Any],
+        callMode: str,
+        latencyMs: int,
+        status: str,
+        message: str,
+    ):
+        metadata = {
+            "functionName": functionName,
+            "argsPassed": argsPassed,
+            "output": output,
+            "callMode": callMode,
+            "latencyMs": latencyMs,
+            "status": status,
+            "message": message,
+        }
+        self.logStep("FUNCTION_CALL", stepName, metadata)
+
+    def logCompressionStep(
+        self,
+        stepName: str,
+        prompt: str,
+        promptTemplate: str,
+        inputs: Dict[str, Any],
+        compressedPrompt: str,
+        inputToken: int,
+        compressedToken: int,
+        outputToken: int,
+        output: str,
+        compressedOutput: str,
+        latencyMs: int,
+        status: str,
+        message: str,
+    ):
+        metadata = {
+            "prompt": prompt,
+            "promptTemplate": promptTemplate,
+            "inputs": inputs,
+            "compressedPrompt": compressedPrompt,
+            "inputToken": inputToken,
+            "compressedToken": compressedToken,
+            "outputToken": outputToken,
+            "output": output,
+            "compressedOutput": compressedOutput,
+            "latencyMs": latencyMs,
+            "status": status,
+            "message": message,
+        }
+        self.logStep("COMPRESSION", stepName, metadata)
+
+    def logCustomScriptStep(
+        self,
+        stepName: str,
+        inputs: Dict[str, Any],
+        script: str,
+        output: str,
+        latencyMs: int,
+        status: str,
+        message: str,
+    ):
+        metadata = {
+            "inputs": inputs,
+            "script": script,
+            "output": output,
+            "latencyMs": latencyMs,
+            "status": status,
+            "message": message,
+        }
+        self.logStep("CUSTOM_SCRIPT", stepName, metadata)

llumo/openai.py

@@ -1,11 +1,27 @@
+import time
 from openai import OpenAI as OpenAIClient
 from .client import LlumoClient
+from .llumoSessionContext import LlumoSessionContext
+from .llumoLogger import LLUMOLogger
+
+
+# evaluation function that uses LlumoClient
+def performEvaluation(data, api_key=None, evals=["Response Correctness"], **kwargs):
+    try:
+        client = LlumoClient(api_key=api_key)
+        results = client.evaluateMultiple(
+            data,
+            evals=evals,
+            createExperiment=kwargs.get("createExperiment", False),
+            playgroundID=kwargs.get("playgroundID"),
+            prompt_template="Give answer to the query: {{query}}, using context: {{context}}",
+            getDataFrame=False,
+        )
+        return results
+    except Exception as e:
+        print(f"Error in perform_evaluation: {e}")
+        raise
 
-# Dummy evaluation function that uses LlumoClient
-def evaluate_multiple(data, api_key=None,evals=["Response Correctness"]):
-    client = LlumoClient(api_key=api_key)
-    results= client.evaluateMultiple(data, evals=evals,createExperiment=False,prompt_template="Give answer to the query: {{query}}, using context: {{context}}",getDataFrame=False)
-    return results
 
 # Wrapper around ChatCompletion to allow custom fields like `.evaluation`
 class ChatCompletionWithEval:
@@ -22,57 +38,187 @@ class ChatCompletionWithEval:
     def __repr__(self):
         return repr(self._response)
 
-class openai(OpenAIClient):
-    def __init__(self, api_key: str):
+
+class OpenAI(OpenAIClient):
+    def __init__(self, api_key: str, session):
         super().__init__(api_key=api_key)
+        self.session = session
+        self.llumo_key = session.apiKey
 
         original_create = self.chat.completions.create
 
-        class ChatCompletionsWrapper:
-            @staticmethod
-            def create(*args, **kwargs):
-                context = kwargs.pop("context", None)
-                evals = kwargs.pop("evals", [])
-                llumo_key = kwargs.pop("llumo_key", None)
-
-                messages = kwargs.get("messages", [])
-                user_message = next(
-                    (m.get("content") for m in reversed(messages) if m.get("role") == "user"),
-                    "",
+        def create_wrapper(*args, **kwargs):
+            context = kwargs.pop("context", None)
+            model = kwargs["model"]
+            create_experiment = kwargs.pop("createExperiment", False)
+
+            messages = kwargs.get("messages", [])
+            user_message = next(
+                (
+                    m.get("content")
+                    for m in reversed(messages)
+                    if m.get("role") == "user"
+                ),
+                "",
+            )
+
+            if not context or context.strip() == "":
+                context = user_message
+
+            # Get IDs from the session logger
+            playground_id = self.session.logger.getPlaygroundID()
+            workspace_id = self.session.logger.getWorkspaceID()
+
+            # Input Bias Evaluation
+            eval_input_bias = [
+                {
+                    "query": user_message,
+                    "context": context,
+                    "output": "",  # No output yet
+                }
+            ]
+            try:
+                start_time = time.time()
+                bias_evaluation_result = performEvaluation(
+                    eval_input_bias,
+                    api_key=self.llumo_key,
+                    evals=["Input Bias"],
+                    playgroundID=playground_id,
+                    workspaceID=workspace_id,
+                    createExperiment=create_experiment,
+                )
+                latency = int((time.time() - start_time) * 1000)
+                # Access the first result object
+                bias_evaluation = bias_evaluation_result[0]
+                # message = "-".join(
+                #     getattr(bias_evaluation, "edgeCases", {}).get("value", [])
+                # )
+                # self.session.logEvalStep(
+                #     stepName=f"EVAL-Input Bias",
+                #     output="",
+                #     context=context,
+                #     query=user_message,
+                #     messageHistory="",
+                #     tools="",
+                #     intermediateSteps="",
+                #     groundTruth="",
+                #     analyticsScore=getattr(bias_evaluation, "analyticsScore", {}),
+                #     reasoning=getattr(bias_evaluation, "reasoning", {}),
+                #     classification=getattr(bias_evaluation, "classification", {}),
+                #     evalLabel=getattr(bias_evaluation, "evalLabel", {}),
+                #     latencyMs=latency,
+                #     status="SUCCESS",
+                #     message=message,
+                # )
+            except Exception as e:
+                print(f"Input Bias evaluation failed: {e}")
+                self.session.logEvalStep(
+                    stepName=f"EVAL-FAILURE",
+                    output="",
+                    context=context,
+                    query=user_message,
+                    messageHistory="",
+                    tools="",
+                    intermediateSteps="",
+                    groundTruth="",
+                    analyticsScore={},
+                    reasoning={},
+                    classification={},
+                    evalLabel={},
+                    latencyMs=0,
+                    status="FAILURE",
+                    message="EVAL_ERROR",
                 )
 
-                # If context is None or empty or whitespace-only, set it to user_message
-                if not context or context.strip() == "":
-                    context = user_message
-
-                response = original_create(*args, **kwargs)
-
-                try:
-                    output_text = response.choices[0].message.content
-                except Exception:
-                    output_text = ""
-
-                eval_input = [{
+            start_time = time.time()
+            response = original_create(*args, **kwargs)
+            latency = int((time.time() - start_time) * 1000)
+            output_text = response.choices[0].message.content
+
+            self.session.logLlmStep(
+                stepName=f"LLM-{user_message[:30]}",
+                model=model,
+                provider="openai",
+                inputTokens=response.usage.prompt_tokens,
+                outputTokens=response.usage.completion_tokens,
+                temperature=kwargs.get("temperature", 0.0),
+                promptTruncated=False,
+                latencyMs=latency,
+                query=user_message,
+                output=output_text,
+                status="SUCCESS",
+                message="",
+            )
+
+            # Response Correctness Evaluation
+            eval_input_correctness = [
+                {
                     "query": user_message,
                     "context": context,
                     "output": output_text,
-                }]
-
-                # Safely call evaluate_multiple, if error return None
-                evaluation = None
-                try:
-                    evaluation = evaluate_multiple(eval_input, api_key=llumo_key,evals=evals)
-                except Exception as e:
-                    # You can optionally log the error here if you want
-                    # print(f"Evaluation failed, skipping: {e}")
-                    evaluation = None
-
-                # If evaluation is None, just return normal response
-                if evaluation is None:
-                    print("Cannot process your request for evaluation, please check your api and try again later.")
-                    return response
-
-                # Otherwise wrap with evaluation attached
-                return ChatCompletionWithEval(response, evaluation)
-
-        self.chat.completions.create = ChatCompletionsWrapper.create
+                }
+            ]
+            try:
+                start_time = time.time()
+                correctness_evaluation_result = performEvaluation(
+                    eval_input_correctness,
+                    api_key=self.llumo_key,
+                    evals=["Response Correctness"],
+                    playgroundID=playground_id,
+                    workspaceID=workspace_id,
+                    createExperiment=create_experiment,
+                )
+                latency = int((time.time() - start_time) * 1000)
+                # Access the first result object
+                correctness_evaluation = correctness_evaluation_result[0]
+                # message = "-".join(
+                #     getattr(correctness_evaluation, "edgeCases", {}).get("value", [])
+                # )
+                # self.session.logEvalStep(
+                #     stepName=f"EVAL-Response Correctness",
+                #     output=output_text,
+                #     context=context,
+                #     query=user_message,
+                #     messageHistory="",
+                #     tools="",
+                #     intermediateSteps="",
+                #     groundTruth="",
+                #     analyticsScore=getattr(
+                #         correctness_evaluation, "analyticsScore", {}
+                #     ),
+                #     reasoning=getattr(correctness_evaluation, "reasoning", {}),
+                #     classification=getattr(
+                #         correctness_evaluation, "classification", {}
+                #     ),
+                #     evalLabel=getattr(correctness_evaluation, "evalLabel", {}),
+                #     latencyMs=latency,
+                #     status="SUCCESS",
+                #     message=message,
+                # )
+            except Exception as e:
+                print(f"Response Correctness evaluation failed: {e}")
+                correctness_evaluation = None
+                self.session.logEvalStep(
+                    stepName=f"EVAL-FAILURE",
+                    output=output_text,
+                    context=context,
+                    query=user_message,
+                    messageHistory="",
+                    tools="",
+                    intermediateSteps="",
+                    groundTruth="",
+                    analyticsScore={},
+                    reasoning={},
+                    classification={},
+                    evalLabel={},
+                    latencyMs=0,
+                    status="FAILURE",
+                    message="EVAL_ERROR",
+                )
+
+            if correctness_evaluation is None:
+                return response
+
+            return ChatCompletionWithEval(response, correctness_evaluation)
+
+        self.chat.completions.create = create_wrapper

{llumo-0.2.24.dist-info → llumo-0.2.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.24
+Version: 0.2.26
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo