llumo 0.2.40__py3-none-any.whl → 0.2.42__py3-none-any.whl

llumo/client.py

@@ -43,9 +43,9 @@ createEvalUrl = "https://backend-api.llumo.ai/api/v1/create-debug-log-for-sdk"
 
 class LlumoClient:
 
-    def __init__(self, api_key, playground_id=None):
+    def __init__(self, api_key, playgroundID=None):
         self.apiKey = api_key
-        self.playgroundID = playground_id
+        self.playgroundID = playgroundID
         self.evalData = []
         self.evals = []
         self.processMapping = {}
@@ -820,6 +820,8 @@ class LlumoClient:
         allBatches = []
         for index, row in dataframe.iterrows():
             # Extract required fields
+            query = row.get("query", "")
+            context = row.get("context", "")
             tools = row.get("tools", "")
             groundTruth = row.get("groundTruth", "")
             messageHistory = row.get("messageHistory", "")
@@ -827,42 +829,43 @@ class LlumoClient:
             output = row.get("output", "")
 
             # Initialize query and context
-            query = ""
-            context = ""
-
-            # Process prompt template if provided
-            if promptTemplate:
-                # Extract template variables
-                keys = re.findall(r"{{(.*?)}}", promptTemplate)
-
-                if not all([key in dataframe.columns for key in keys]):
-                    raise LlumoAIError.InvalidPromptTemplate()
-
-                # Populate template and separate query/context
-                populated_template = promptTemplate
-                for key in keys:
-                    value = row.get(key, "")
-                    if isinstance(value, str):
-                        length = len(value.split()) * 1.5
-                        if length <= 50:
-                            # Short value - include in query via template
-                            temp_obj = {key: value}
-                            populated_template = getInputPopulatedPrompt(populated_template, temp_obj)
-                        else:
-                            # Long value - add to context
-                            context += f" {key}: {value}, "
-
-                query = populated_template.strip()
-
-                # Add any remaining context from other fields
-                if not context.strip():
-                    for key, value in row.items():
-                        if key not in keys and isinstance(value, str) and value.strip():
-                            context += f" {key}: {value}, "
-            else:
-                # No prompt template - use direct query and context fields
-                query = row.get("query", "")
-                context = row.get("context", "")
+            # query = ""
+            # context = ""
+
+            # # Process prompt template if provided
+            # if promptTemplate:
+            #     # Extract template variables
+            #     keys = re.findall(r"{{(.*?)}}", promptTemplate)
+
+            #     if not all([key in dataframe.columns for key in keys]):
+            #         # raise LlumoAIError.InvalidPromptTemplate()
+            #         break 
+                    
+            #     # Populate template and separate query/context
+            #     populated_template = promptTemplate
+            #     for key in keys:
+            #         value = row.get(key, "")
+            #         if isinstance(value, str):
+            #             length = len(value.split()) * 1.5
+            #             if length <= 50:
+            #                 # Short value - include in query via template
+            #                 temp_obj = {key: value}
+            #                 populated_template = getInputPopulatedPrompt(populated_template, temp_obj)
+            #             else:
+            #                 # Long value - add to context
+            #                 context += f" {key}: {value}, "
+
+            #     query = populated_template.strip()
+
+            #     # Add any remaining context from other fields
+            #     if not context.strip():
+            #         for key, value in row.items():
+            #             if key not in keys and isinstance(value, str) and value.strip():
+            #                 context += f" {key}: {value}, "
+            # else:
+            #     # No prompt template - use direct query and context fields
+            #     query = row.get("query", "")
+            #     context = row.get("context", "")
 
             INPUT_TOKEN_PRICE = 0.0000025
             OUTPUT_TOKEN_PRICE = 0.00001
@@ -885,7 +888,10 @@ class LlumoClient:
                  "totalTokens": totalTokens,
                  "cost": round(cost, 8),
                  "modelsUsed": "gpt-4o",
-                 "latency":round(random.uniform(1,1.6),2)
+                 "latency":round(random.uniform(1,1.6),2),
+                "promptTemplate": promptTemplate,
+                "systemInstructions": systemInstructions
+
 
             }
             currentTime = datetime(2025, 8, 2, 10, 20, 15, tzinfo=timezone.utc)
@@ -910,8 +916,7 @@ class LlumoClient:
                 "columnID":rowID,
                 "rowID":columnID,
                 "latency": random.randint(1000, 1500),
-                "promptTemplate":promptTemplate,
-                "systemInstructions":systemInstructions
+
 
             }
 
@@ -921,7 +926,7 @@ class LlumoClient:
         for i, batch in enumerate(allBatches, start=1):
 
             try:
-                # print(batch)
+                #  print(batch)
                 response = postForDebugLogs(record=batch,workspaceID=workspaceID)
 
                 # failure case inside response
@@ -938,9 +943,6 @@ class LlumoClient:
 
         print("Records Uploaded successfully. You may now review your logs at: https://app.llumo.ai/logs")
 
-
-        # Wait for results
-
     # def evaluateMultiple(
     #     self,
     #     data,
@@ -2082,6 +2084,115 @@ class LlumoClient:
                     definationMapping=self.definationMapping,
                 )
 
+    def getEvaluateMultiple(
+        self,
+        data,
+        evals
+    ) -> List:
+
+        # print("Evaluating multiple data with evals:", data, evals)
+
+        dataID = uuid.uuid4().hex[:36]
+
+        self.validateApiKey()
+
+        if not self.workspaceID:
+            raise LlumoAIError("Workspace ID not found after validation.")
+
+        payload = {
+            "dataID": dataID,
+            "data": data,
+            "evals": evals,
+            "workspaceID": self.workspaceID,
+            "playgroundID": self.playgroundID,
+        }
+
+        #  print("payload", payload)
+
+        # Create evaluation
+        requests.post(
+            "https://backend-api.llumo.ai/api/v1/sdk/create-evaluation-Multiple",
+            json=payload,
+            headers={
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {self.apiKey}",
+            },
+        )
+
+        final_result_data = []
+
+        cursor = "0-0"
+        limit = 10
+        all_data_fetched = False
+
+        while not all_data_fetched:
+            try:
+                response = requests.get(
+                    "https://backend-api.llumo.ai/api/v1/sdk/poll",
+                    params={
+                        "cursor": cursor,
+                        "dataID": dataID,
+                        "limit": limit,
+                    },
+                )
+
+                response_data = response.json()
+                result_data = response_data.get("debugLog", {})
+                print("resultData", result_data)
+
+                results = result_data.get("results", [])
+                final_result_data.extend(results)
+
+                cursor = result_data.get("nextCursor")
+
+                if len(final_result_data) == len(data):
+                    all_data_fetched = True
+
+                time.sleep(10)
+
+            except Exception as error:
+                print("error", error)
+                all_data_fetched = True
+
+        # Shape results
+        formatted_results = []
+
+        for row in final_result_data:
+            score: Dict[str, float | None] = {}
+            reasoning: Dict[str, str] = {}
+
+            for eval_name in evals:
+                details = row.get(eval_name)
+
+                if isinstance(details, dict):
+                    if isinstance(details.get("value"), (int, float)):
+                        score[eval_name] = details.get("value")
+                    else:
+                        score[eval_name] = details.get("score")
+                    reasoning[eval_name] = details.get("reasoning", "")
+
+                elif "score" in row:
+                    score[eval_name] = (
+                        row["score"] if isinstance(row["score"], (int, float)) else None
+                    )
+                    reasoning[eval_name] = row.get("reasoning", "")
+                else:
+                    score[eval_name] = None
+                    reasoning[eval_name] = ""
+
+            formatted_row = {
+                "context": row.get("context", ""),
+                "query": row.get("query", ""),
+                "output": row.get("output", ""),
+                "score": score,
+                "reasoning": reasoning,
+            }
+
+            # print(formatted_row)
+            formatted_results.append(formatted_row)
+
+        return formatted_results
+
 
 class SafeDict(dict):
     def __missing__(self, key):

llumo/llumoSessionContext.py

@@ -30,7 +30,7 @@ def getLlumoRun():
 
 class LlumoSessionContext(LlumoClient):
     def __init__(self, logger, sessionID: Optional[str] = None):
-        super().__init__(api_key=logger.apiKey, playground_id=logger.getPlaygroundID())
+        super().__init__(api_key=logger.apiKey, playgroundID=logger.getPlaygroundID())
         self.sessionID = sessionID or str(uuid.uuid4().hex[:14])
         self.logger = logger
         self.apiKey = logger.apiKey
@@ -102,9 +102,10 @@ class LlumoSessionContext(LlumoClient):
                 "totalTokens": "",
                 "cost": "",
                 "modelsUsed": "gpt-4o",
+                "promptTemplate": promptTemplate,
+                "systemInstructions": systemInstructions
                        },
-            "promptTemplate":promptTemplate,
-            "systemInstructions":systemInstructions
+
         }
 
         self.threadLlumoRun = _ctxLlumoRun.set(llumoRun)

{llumo-0.2.40.dist-info → llumo-0.2.42.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: llumo
-Version: 0.2.40
+Version: 0.2.42
 Summary: Python SDK for interacting with the Llumo ai API.
 Home-page: https://www.llumo.ai/
 Author: Llumo

{llumo-0.2.40.dist-info → llumo-0.2.42.dist-info}/RECORD

@@ -2,19 +2,19 @@ llumo/__init__.py,sha256=kkuppu7ZPiVZFdnYzJ9BM3syMbYHOSZLpwKwAvGHsnY,311
 llumo/callback.py,sha256=XQbImLnd64_B2Iir-FCaJcL5Kmpm3RlXJH2zDRXk_yk,25135
 llumo/callbacks-0.py,sha256=TEIOCWRvk2UYsTmBMBsnlgpqWvr-2y3a6d0w_e96NRM,8958
 llumo/chains.py,sha256=6lCgLseh04RUgc6SahhmvQj82quay2Mi1j8gPUlx8Es,2923
-llumo/client.py,sha256=W0_i5M5mN0B_YfDWaT6qDUGY93PeqzcK6JxZdIFXibk,79903
+llumo/client.py,sha256=Sf9QaKWIsAHYAX78YQT2U93A0LuwkluKQQvCMg2vRwM,83386
 llumo/exceptions.py,sha256=1OyhN9YL9LcyUPUsqYHq6Rret0udATZAwMVJaio2_Ec,2123
 llumo/execution.py,sha256=nWbJ7AvWuUPcOb6i-JzKRna_PvF-ewZTiK8skS-5n3w,1380
 llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
 llumo/google.py,sha256=6y9YnDFDRHv6-sQNT5LIsV9p31BCN0B9eow5KTRBWfM,2185
 llumo/helpingFuntions.py,sha256=eMR2Rq8vw4X5sIESOvjOBrEvyYE00Eq7XlQjV66eVcg,29977
 llumo/llumoLogger.py,sha256=ALM4461jItWcvYL9HzTGmB-X-M77YF_9PTeTPxtmP_E,2223
-llumo/llumoSessionContext.py,sha256=f4C6w5H7Fym6lC6KK8wY8kW5VPb0hNrBEvh6zkCSkfk,14881
+llumo/llumoSessionContext.py,sha256=QaiQtB13dITTVINpzEAEVlO-fpW362XkgMjM1W4EHoo,14891
 llumo/models.py,sha256=aVEZsOOoQx5LeNtwSyBxqvrINq0izH3QWu_YjsMPE6o,2910
 llumo/openai.py,sha256=VstBzaORe8Tq0feUIIEszzcN1oq6TJfkPviaCr5d3Bw,8950
 llumo/sockets.py,sha256=pfWz1zTEiwqJhdbSy3i3_Y4WlIdJ3cuac11wMePeBS0,6130
-llumo-0.2.40.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
-llumo-0.2.40.dist-info/METADATA,sha256=qzr5dj9AT4vbOpU_IgwHpFcqojjupUusLIoBqYR-qXo,1662
-llumo-0.2.40.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-llumo-0.2.40.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
-llumo-0.2.40.dist-info/RECORD,,
+llumo-0.2.42.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
+llumo-0.2.42.dist-info/METADATA,sha256=v5zWRFLNSHoMFuXVY3kgdaAcsj3L16AWmTQH6sFEs_k,1662
+llumo-0.2.42.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+llumo-0.2.42.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
+llumo-0.2.42.dist-info/RECORD,,