llumo 0.2.41__py3-none-any.whl → 0.2.43__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llumo/client.py +396 -53
- llumo/exceptions.py +9 -0
- llumo/helpingFuntions.py +106 -0
- llumo/llumoLogger.py +3 -2
- llumo/llumoSessionContext.py +3 -2
- {llumo-0.2.41.dist-info → llumo-0.2.43.dist-info}/METADATA +1 -1
- {llumo-0.2.41.dist-info → llumo-0.2.43.dist-info}/RECORD +10 -10
- {llumo-0.2.41.dist-info → llumo-0.2.43.dist-info}/WHEEL +0 -0
- {llumo-0.2.41.dist-info → llumo-0.2.43.dist-info}/licenses/LICENSE +0 -0
- {llumo-0.2.41.dist-info → llumo-0.2.43.dist-info}/top_level.txt +0 -0
llumo/client.py
CHANGED
|
@@ -8,10 +8,12 @@ import json
|
|
|
8
8
|
import uuid
|
|
9
9
|
import warnings
|
|
10
10
|
import os
|
|
11
|
-
|
|
11
|
+
from typing import List, Dict, Optional
|
|
12
12
|
import itertools
|
|
13
13
|
import pandas as pd
|
|
14
|
-
from typing import List, Dict
|
|
14
|
+
from typing import List, Dict, Any
|
|
15
|
+
|
|
16
|
+
|
|
15
17
|
from .models import AVAILABLEMODELS, getProviderFromModel, Provider
|
|
16
18
|
from .execution import ModelExecutor
|
|
17
19
|
from .exceptions import LlumoAIError
|
|
@@ -50,6 +52,7 @@ class LlumoClient:
|
|
|
50
52
|
self.evals = []
|
|
51
53
|
self.processMapping = {}
|
|
52
54
|
self.definationMapping = {}
|
|
55
|
+
self.ALL_USER_AIM = ['incorrectOutput', 'incorrectInput', 'hallucination', 'ragQuality', 'contextMismanagement', 'toolCallIssues', 'agentReasoning', 'stuckAgents', 'jsonErrors', 'highLatency', 'highCost', 'safetyBlocks', 'modelRouting', 'systemErrors', 'promptAdherence']
|
|
53
56
|
|
|
54
57
|
def validateApiKey(self, evalName="Input Bias"):
|
|
55
58
|
headers = {
|
|
@@ -780,7 +783,9 @@ class LlumoClient:
|
|
|
780
783
|
self,
|
|
781
784
|
data,
|
|
782
785
|
promptTemplate="",
|
|
783
|
-
systemInstructions = ""
|
|
786
|
+
systemInstructions = "",
|
|
787
|
+
multiTurnChat=False,
|
|
788
|
+
createMultipleLogs = True
|
|
784
789
|
|
|
785
790
|
):
|
|
786
791
|
if isinstance(data, dict):
|
|
@@ -788,7 +793,7 @@ class LlumoClient:
|
|
|
788
793
|
elif not isinstance(data, list):
|
|
789
794
|
raise ValueError("Data should be a dict or a list of dicts.")
|
|
790
795
|
|
|
791
|
-
dataframe = pd.DataFrame(data).astype(str)
|
|
796
|
+
dataframe = pd.DataFrame(data).astype(str).replace(to_replace="nan",value = "")
|
|
792
797
|
workspaceID = None
|
|
793
798
|
email = None
|
|
794
799
|
|
|
@@ -820,49 +825,72 @@ class LlumoClient:
|
|
|
820
825
|
allBatches = []
|
|
821
826
|
for index, row in dataframe.iterrows():
|
|
822
827
|
# Extract required fields
|
|
828
|
+
query = row.get("query", "")
|
|
829
|
+
context = row.get("context", "")
|
|
830
|
+
|
|
823
831
|
tools = row.get("tools", "")
|
|
824
832
|
groundTruth = row.get("groundTruth", "")
|
|
825
|
-
messageHistory = row.get("messageHistory", "")
|
|
826
|
-
intermediateSteps = row.get("intermediateSteps", "")
|
|
827
|
-
output = row.get("output", "")
|
|
828
833
|
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
834
|
+
if multiTurnChat==False:
|
|
835
|
+
# ---- SINGLE TURN (existing behavior) ----
|
|
836
|
+
messageHistory = row.get("messageHistory", "")
|
|
832
837
|
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
keys = re.findall(r"{{(.*?)}}", promptTemplate)
|
|
838
|
+
else:
|
|
839
|
+
# ---- MULTI TURN ----
|
|
840
|
+
multiTurnData = createMessageHistory(data, index)
|
|
837
841
|
|
|
838
|
-
if
|
|
839
|
-
|
|
842
|
+
if createMultipleLogs==True:
|
|
843
|
+
# each row will get history till that point
|
|
844
|
+
messageHistory = multiTurnData
|
|
845
|
+
else:
|
|
846
|
+
# only final API call should contain full history
|
|
847
|
+
if index == len(dataframe) - 1:
|
|
848
|
+
messageHistory = multiTurnData
|
|
849
|
+
else:
|
|
850
|
+
messageHistory = ""
|
|
840
851
|
|
|
841
|
-
# Populate template and separate query/context
|
|
842
|
-
populated_template = promptTemplate
|
|
843
|
-
for key in keys:
|
|
844
|
-
value = row.get(key, "")
|
|
845
|
-
if isinstance(value, str):
|
|
846
|
-
length = len(value.split()) * 1.5
|
|
847
|
-
if length <= 50:
|
|
848
|
-
# Short value - include in query via template
|
|
849
|
-
temp_obj = {key: value}
|
|
850
|
-
populated_template = getInputPopulatedPrompt(populated_template, temp_obj)
|
|
851
|
-
else:
|
|
852
|
-
# Long value - add to context
|
|
853
|
-
context += f" {key}: {value}, "
|
|
854
852
|
|
|
855
|
-
query = populated_template.strip()
|
|
856
853
|
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
854
|
+
intermediateSteps = row.get("intermediateSteps", "")
|
|
855
|
+
output = row.get("output", "")
|
|
856
|
+
|
|
857
|
+
# # Initialize query and context
|
|
858
|
+
# query = ""
|
|
859
|
+
# context = ""
|
|
860
|
+
#
|
|
861
|
+
# # Process prompt template if provided
|
|
862
|
+
# if promptTemplate:
|
|
863
|
+
# # Extract template variables
|
|
864
|
+
# keys = re.findall(r"{{(.*?)}}", promptTemplate)
|
|
865
|
+
#
|
|
866
|
+
# if not all([key in dataframe.columns for key in keys]):
|
|
867
|
+
# raise LlumoAIError.InvalidPromptTemplate()
|
|
868
|
+
#
|
|
869
|
+
# # Populate template and separate query/context
|
|
870
|
+
# populated_template = promptTemplate
|
|
871
|
+
# for key in keys:
|
|
872
|
+
# value = row.get(key, "")
|
|
873
|
+
# if isinstance(value, str):
|
|
874
|
+
# length = len(value.split()) * 1.5
|
|
875
|
+
# if length <= 50:
|
|
876
|
+
# # Short value - include in query via template
|
|
877
|
+
# temp_obj = {key: value}
|
|
878
|
+
# populated_template = getInputPopulatedPrompt(populated_template, temp_obj)
|
|
879
|
+
# else:
|
|
880
|
+
# # Long value - add to context
|
|
881
|
+
# context += f" {key}: {value}, "
|
|
882
|
+
#
|
|
883
|
+
# query = populated_template.strip()
|
|
884
|
+
#
|
|
885
|
+
# # Add any remaining context from other fields
|
|
886
|
+
# if not context.strip():
|
|
887
|
+
# for key, value in row.items():
|
|
888
|
+
# if key not in keys and isinstance(value, str) and value.strip():
|
|
889
|
+
# context += f" {key}: {value}, "
|
|
890
|
+
# else:
|
|
891
|
+
# # No prompt template - use direct query and context fields
|
|
892
|
+
# query = row.get("query", "")
|
|
893
|
+
# context = row.get("context", "")
|
|
866
894
|
|
|
867
895
|
INPUT_TOKEN_PRICE = 0.0000025
|
|
868
896
|
OUTPUT_TOKEN_PRICE = 0.00001
|
|
@@ -885,9 +913,13 @@ class LlumoClient:
|
|
|
885
913
|
"totalTokens": totalTokens,
|
|
886
914
|
"cost": round(cost, 8),
|
|
887
915
|
"modelsUsed": "gpt-4o",
|
|
888
|
-
"latency":round(random.uniform(1,1.6),2)
|
|
916
|
+
"latency":round(random.uniform(1,1.6),2),
|
|
917
|
+
"promptTemplate": promptTemplate,
|
|
918
|
+
"systemInstructions": systemInstructions
|
|
919
|
+
|
|
889
920
|
|
|
890
921
|
}
|
|
922
|
+
|
|
891
923
|
currentTime = datetime(2025, 8, 2, 10, 20, 15, tzinfo=timezone.utc)
|
|
892
924
|
createdAt = currentTime.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
|
893
925
|
rowID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
|
|
@@ -910,8 +942,7 @@ class LlumoClient:
|
|
|
910
942
|
"columnID":rowID,
|
|
911
943
|
"rowID":columnID,
|
|
912
944
|
"latency": random.randint(1000, 1500),
|
|
913
|
-
|
|
914
|
-
"systemInstructions":systemInstructions
|
|
945
|
+
|
|
915
946
|
|
|
916
947
|
}
|
|
917
948
|
|
|
@@ -2082,14 +2113,147 @@ class LlumoClient:
|
|
|
2082
2113
|
definationMapping=self.definationMapping,
|
|
2083
2114
|
)
|
|
2084
2115
|
|
|
2085
|
-
def get_evaluate_multiple(
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
) -> List:
|
|
2116
|
+
# def get_evaluate_multiple(
|
|
2117
|
+
# self,
|
|
2118
|
+
# data,
|
|
2119
|
+
# evals
|
|
2120
|
+
# ) -> List:
|
|
2121
|
+
|
|
2122
|
+
# print("Evaluating multiple data with evals:", data, evals)
|
|
2090
2123
|
|
|
2091
|
-
|
|
2124
|
+
# dataID = uuid.uuid4().hex[:36]
|
|
2092
2125
|
|
|
2126
|
+
# self.validateApiKey()
|
|
2127
|
+
|
|
2128
|
+
# if not self.workspaceID:
|
|
2129
|
+
# raise LlumoAIError("Workspace ID not found after validation.")
|
|
2130
|
+
|
|
2131
|
+
# payload = {
|
|
2132
|
+
# "dataID": dataID,
|
|
2133
|
+
# "data": data,
|
|
2134
|
+
# "evals": evals,
|
|
2135
|
+
# "workspaceID": self.workspaceID,
|
|
2136
|
+
# "playgroundID": self.playgroundID,
|
|
2137
|
+
# }
|
|
2138
|
+
|
|
2139
|
+
# print("payload", payload)
|
|
2140
|
+
|
|
2141
|
+
# # Create evaluation
|
|
2142
|
+
# requests.post(
|
|
2143
|
+
# "https://backend-api.llumo.ai/api/v1/sdk/create-evaluation-Multiple",
|
|
2144
|
+
# json=payload,
|
|
2145
|
+
# headers={
|
|
2146
|
+
# "Content-Type": "application/json",
|
|
2147
|
+
# "Authorization": f"Bearer {self.apiKey}",
|
|
2148
|
+
# },
|
|
2149
|
+
# )
|
|
2150
|
+
|
|
2151
|
+
# final_result_data = []
|
|
2152
|
+
|
|
2153
|
+
# cursor = "0-0"
|
|
2154
|
+
# limit = 10
|
|
2155
|
+
# all_data_fetched = False
|
|
2156
|
+
|
|
2157
|
+
# while not all_data_fetched:
|
|
2158
|
+
# try:
|
|
2159
|
+
# response = requests.get(
|
|
2160
|
+
# "https://backend-api.llumo.ai/api/v1/sdk/poll",
|
|
2161
|
+
# params={
|
|
2162
|
+
# "cursor": cursor,
|
|
2163
|
+
# "dataID": dataID,
|
|
2164
|
+
# "limit": limit,
|
|
2165
|
+
# },
|
|
2166
|
+
# )
|
|
2167
|
+
|
|
2168
|
+
# response_data = response.json()
|
|
2169
|
+
# result_data = response_data.get("debugLog", {})
|
|
2170
|
+
# print("resultData", result_data)
|
|
2171
|
+
|
|
2172
|
+
# results = result_data.get("results", [])
|
|
2173
|
+
# final_result_data.extend(results)
|
|
2174
|
+
|
|
2175
|
+
# cursor = result_data.get("nextCursor")
|
|
2176
|
+
|
|
2177
|
+
# if len(final_result_data) == len(data):
|
|
2178
|
+
# all_data_fetched = True
|
|
2179
|
+
|
|
2180
|
+
# time.sleep(10)
|
|
2181
|
+
|
|
2182
|
+
# except Exception as error:
|
|
2183
|
+
# print("error", error)
|
|
2184
|
+
# all_data_fetched = True
|
|
2185
|
+
|
|
2186
|
+
# # Shape results
|
|
2187
|
+
# formatted_results = []
|
|
2188
|
+
|
|
2189
|
+
# for row in final_result_data:
|
|
2190
|
+
# score: Dict[str, float | None] = {}
|
|
2191
|
+
# reasoning: Dict[str, str] = {}
|
|
2192
|
+
|
|
2193
|
+
# for eval_name in evals:
|
|
2194
|
+
# details = row.get(eval_name)
|
|
2195
|
+
|
|
2196
|
+
# if isinstance(details, dict):
|
|
2197
|
+
# if isinstance(details.get("value"), (int, float)):
|
|
2198
|
+
# score[eval_name] = details.get("value")
|
|
2199
|
+
# else:
|
|
2200
|
+
# score[eval_name] = details.get("score")
|
|
2201
|
+
# reasoning[eval_name] = details.get("reasoning", "")
|
|
2202
|
+
|
|
2203
|
+
# elif "score" in row:
|
|
2204
|
+
# score[eval_name] = (
|
|
2205
|
+
# row["score"] if isinstance(row["score"], (int, float)) else None
|
|
2206
|
+
# )
|
|
2207
|
+
# reasoning[eval_name] = row.get("reasoning", "")
|
|
2208
|
+
# else:
|
|
2209
|
+
# score[eval_name] = None
|
|
2210
|
+
# reasoning[eval_name] = ""
|
|
2211
|
+
|
|
2212
|
+
# formatted_row = {
|
|
2213
|
+
# "context": row.get("context", ""),
|
|
2214
|
+
# "query": row.get("query", ""),
|
|
2215
|
+
# "output": row.get("output", ""),
|
|
2216
|
+
# "score": score,
|
|
2217
|
+
# "reasoning": reasoning,
|
|
2218
|
+
# }
|
|
2219
|
+
|
|
2220
|
+
# print(formatted_row)
|
|
2221
|
+
# formatted_results.append(formatted_row)
|
|
2222
|
+
|
|
2223
|
+
# return formatted_results
|
|
2224
|
+
|
|
2225
|
+
def getEvaluateMultiple(
|
|
2226
|
+
self,
|
|
2227
|
+
data,
|
|
2228
|
+
evals,
|
|
2229
|
+
promptTemplate="",
|
|
2230
|
+
systemInstructions="",
|
|
2231
|
+
multiTurnChat=False,
|
|
2232
|
+
createMultipleLogs=True
|
|
2233
|
+
):
|
|
2234
|
+
# print("Evaluating multiple data with evals:", data, evals)
|
|
2235
|
+
try:
|
|
2236
|
+
self.validateApiKey()
|
|
2237
|
+
except Exception as e:
|
|
2238
|
+
print(f"Error during API key validation: {str(e)}")
|
|
2239
|
+
if hasattr(e, "response") and getattr(e, "response", None) is not None:
|
|
2240
|
+
print(f"Status code: {e.response.status_code}")
|
|
2241
|
+
print(f"Response content: {e.response.text[:500]}...")
|
|
2242
|
+
raise
|
|
2243
|
+
|
|
2244
|
+
userHits = checkUserHits(
|
|
2245
|
+
self.workspaceID,
|
|
2246
|
+
self.hasSubscribed,
|
|
2247
|
+
self.trialEndDate,
|
|
2248
|
+
self.subscriptionEndDate,
|
|
2249
|
+
self.hitsAvailable,
|
|
2250
|
+
len(data),
|
|
2251
|
+
)
|
|
2252
|
+
|
|
2253
|
+
if not userHits["success"]:
|
|
2254
|
+
raise LlumoAIError.InsufficientCredits(userHits["message"])
|
|
2255
|
+
|
|
2256
|
+
print("✅ SDK integration successful!")
|
|
2093
2257
|
dataID = uuid.uuid4().hex[:36]
|
|
2094
2258
|
|
|
2095
2259
|
self.validateApiKey()
|
|
@@ -2097,6 +2261,38 @@ class LlumoClient:
|
|
|
2097
2261
|
if not self.workspaceID:
|
|
2098
2262
|
raise LlumoAIError("Workspace ID not found after validation.")
|
|
2099
2263
|
|
|
2264
|
+
if promptTemplate:
|
|
2265
|
+
for row in data:
|
|
2266
|
+
row["promptTemplate"] = promptTemplate
|
|
2267
|
+
if systemInstructions:
|
|
2268
|
+
for row in data:
|
|
2269
|
+
row["systemInstructions"] = systemInstructions
|
|
2270
|
+
|
|
2271
|
+
if multiTurnChat == True:
|
|
2272
|
+
if createMultipleLogs == True:
|
|
2273
|
+
dataWithMessageHistory = []
|
|
2274
|
+
for indx, row in enumerate(data):
|
|
2275
|
+
messageHistory = createMessageHistory(data, currentIndex=indx)
|
|
2276
|
+
rowCopy = row.copy()
|
|
2277
|
+
rowCopy["messageHistory"] = messageHistory
|
|
2278
|
+
dataWithMessageHistory.append(rowCopy)
|
|
2279
|
+
data = dataWithMessageHistory
|
|
2280
|
+
else:
|
|
2281
|
+
dataWithMessageHistory = []
|
|
2282
|
+
for indx, row in enumerate(data):
|
|
2283
|
+
if indx == len(data) - 1:
|
|
2284
|
+
messageHistory = createMessageHistory(data, currentIndex=indx)
|
|
2285
|
+
rowCopy = row.copy()
|
|
2286
|
+
rowCopy["messageHistory"] = messageHistory
|
|
2287
|
+
dataWithMessageHistory.append(rowCopy)
|
|
2288
|
+
else:
|
|
2289
|
+
row["messageHistory"] = ""
|
|
2290
|
+
dataWithMessageHistory.append(row)
|
|
2291
|
+
|
|
2292
|
+
data = dataWithMessageHistory
|
|
2293
|
+
|
|
2294
|
+
# print("DATA:")
|
|
2295
|
+
# print(data)
|
|
2100
2296
|
payload = {
|
|
2101
2297
|
"dataID": dataID,
|
|
2102
2298
|
"data": data,
|
|
@@ -2105,7 +2301,7 @@ class LlumoClient:
|
|
|
2105
2301
|
"playgroundID": self.playgroundID,
|
|
2106
2302
|
}
|
|
2107
2303
|
|
|
2108
|
-
print("payload", payload)
|
|
2304
|
+
# print("payload", payload)
|
|
2109
2305
|
|
|
2110
2306
|
# Create evaluation
|
|
2111
2307
|
requests.post(
|
|
@@ -2123,6 +2319,7 @@ class LlumoClient:
|
|
|
2123
2319
|
limit = 10
|
|
2124
2320
|
all_data_fetched = False
|
|
2125
2321
|
|
|
2322
|
+
print("✅ Evaluation Started...")
|
|
2126
2323
|
while not all_data_fetched:
|
|
2127
2324
|
try:
|
|
2128
2325
|
response = requests.get(
|
|
@@ -2136,7 +2333,7 @@ class LlumoClient:
|
|
|
2136
2333
|
|
|
2137
2334
|
response_data = response.json()
|
|
2138
2335
|
result_data = response_data.get("debugLog", {})
|
|
2139
|
-
print("resultData", result_data)
|
|
2336
|
+
# print("resultData", result_data)
|
|
2140
2337
|
|
|
2141
2338
|
results = result_data.get("results", [])
|
|
2142
2339
|
final_result_data.extend(results)
|
|
@@ -2179,17 +2376,163 @@ class LlumoClient:
|
|
|
2179
2376
|
reasoning[eval_name] = ""
|
|
2180
2377
|
|
|
2181
2378
|
formatted_row = {
|
|
2182
|
-
"context": row.get("context", ""),
|
|
2183
2379
|
"query": row.get("query", ""),
|
|
2380
|
+
"context": row.get("context", ""),
|
|
2184
2381
|
"output": row.get("output", ""),
|
|
2185
2382
|
"score": score,
|
|
2186
2383
|
"reasoning": reasoning,
|
|
2187
2384
|
}
|
|
2188
2385
|
|
|
2189
|
-
print(formatted_row)
|
|
2386
|
+
# print(formatted_row)
|
|
2190
2387
|
formatted_results.append(formatted_row)
|
|
2191
2388
|
|
|
2192
|
-
return formatted_results
|
|
2389
|
+
return {"llumoEval": formatted_results}
|
|
2390
|
+
|
|
2391
|
+
|
|
2392
|
+
def getInsights(self,logs:List,userAim:List[str],promptTemplate:str = ""
|
|
2393
|
+
,systemInstructions:str="",multiTurnChat=False,createMultipleLogs=True):
|
|
2394
|
+
|
|
2395
|
+
try:
|
|
2396
|
+
self.validateApiKey()
|
|
2397
|
+
except Exception as e:
|
|
2398
|
+
print(f"Error during API key validation: {str(e)}")
|
|
2399
|
+
if hasattr(e, "response") and getattr(e, "response", None) is not None:
|
|
2400
|
+
print(f"Status code: {e.response.status_code}")
|
|
2401
|
+
print(f"Response content: {e.response.text[:500]}...")
|
|
2402
|
+
raise
|
|
2403
|
+
|
|
2404
|
+
userHits = checkUserHits(
|
|
2405
|
+
self.workspaceID,
|
|
2406
|
+
self.hasSubscribed,
|
|
2407
|
+
self.trialEndDate,
|
|
2408
|
+
self.subscriptionEndDate,
|
|
2409
|
+
self.hitsAvailable,
|
|
2410
|
+
1,
|
|
2411
|
+
)
|
|
2412
|
+
|
|
2413
|
+
if not userHits["success"]:
|
|
2414
|
+
raise LlumoAIError.InsufficientCredits(userHits["message"])
|
|
2415
|
+
|
|
2416
|
+
if len(logs)==0 :
|
|
2417
|
+
raise LlumoAIError.emptyLogList()
|
|
2418
|
+
|
|
2419
|
+
if not isinstance(userAim, list):
|
|
2420
|
+
raise TypeError(f"userAim must be list, got {type(userAim).__name__}")
|
|
2421
|
+
|
|
2422
|
+
if any(aim not in self.ALL_USER_AIM for aim in userAim):
|
|
2423
|
+
errorMessage = f"Please pass a valid user aim. Only acceptable user aims are->{self.ALL_USER_AIM}"
|
|
2424
|
+
raise LlumoAIError.invalidUserAim(details=errorMessage)
|
|
2425
|
+
|
|
2426
|
+
|
|
2427
|
+
|
|
2428
|
+
if multiTurnChat == True:
|
|
2429
|
+
if createMultipleLogs == True:
|
|
2430
|
+
dataWithMessageHistory = []
|
|
2431
|
+
for indx, row in enumerate(logs):
|
|
2432
|
+
messageHistory = createMessageHistory(logs, currentIndex=indx)
|
|
2433
|
+
rowCopy = row.copy()
|
|
2434
|
+
rowCopy["messageHistory"] = messageHistory
|
|
2435
|
+
dataWithMessageHistory.append(rowCopy)
|
|
2436
|
+
logs= dataWithMessageHistory
|
|
2437
|
+
else:
|
|
2438
|
+
dataWithMessageHistory = []
|
|
2439
|
+
for indx, row in enumerate(logs):
|
|
2440
|
+
if indx == len(logs) - 1:
|
|
2441
|
+
messageHistory = createMessageHistory(logs, currentIndex=indx)
|
|
2442
|
+
rowCopy = row.copy()
|
|
2443
|
+
rowCopy["messageHistory"] = messageHistory
|
|
2444
|
+
dataWithMessageHistory.append(rowCopy)
|
|
2445
|
+
else:
|
|
2446
|
+
row["messageHistory"] = ""
|
|
2447
|
+
dataWithMessageHistory.append(row)
|
|
2448
|
+
logs = dataWithMessageHistory
|
|
2449
|
+
|
|
2450
|
+
if (promptTemplate!="") or systemInstructions!="":
|
|
2451
|
+
logs = addPromptAndInstructionInLogs(logData = logs ,promptTemplate=promptTemplate,systemInstruction=systemInstructions)
|
|
2452
|
+
|
|
2453
|
+
# print("[LOGS: ]")
|
|
2454
|
+
# print(logs)
|
|
2455
|
+
|
|
2456
|
+
# 1. Create Report
|
|
2457
|
+
print("✅ Generating Insights Now....")
|
|
2458
|
+
dataID = uuid.uuid4().hex[:36]
|
|
2459
|
+
try:
|
|
2460
|
+
payload = {
|
|
2461
|
+
"data":logs,
|
|
2462
|
+
"userAim":userAim,
|
|
2463
|
+
"dataID":dataID
|
|
2464
|
+
}
|
|
2465
|
+
create_url = "https://backend-api.llumo.ai/api/v1/sdk/create-insight-report"
|
|
2466
|
+
response = requests.post(create_url, json=payload)
|
|
2467
|
+
|
|
2468
|
+
# Check if request was successful
|
|
2469
|
+
response.raise_for_status()
|
|
2470
|
+
|
|
2471
|
+
# print(f"Create Response: {response.json()}")
|
|
2472
|
+
except requests.exceptions.RequestException as e:
|
|
2473
|
+
# Check for response data in the error
|
|
2474
|
+
error_data = e.response.json() if e.response else str(e)
|
|
2475
|
+
print(f"Error in create request: {error_data}")
|
|
2476
|
+
return
|
|
2477
|
+
|
|
2478
|
+
# 2. Poll for Results
|
|
2479
|
+
# print("\n--- Starting Polling ---")
|
|
2480
|
+
cursor = '0-0'
|
|
2481
|
+
is_complete = False
|
|
2482
|
+
max_polls = 20
|
|
2483
|
+
poll_count = 0
|
|
2484
|
+
|
|
2485
|
+
insight_result = []
|
|
2486
|
+
while not is_complete and poll_count < max_polls:
|
|
2487
|
+
poll_count += 1
|
|
2488
|
+
|
|
2489
|
+
try:
|
|
2490
|
+
poll_params = {
|
|
2491
|
+
'dataID': dataID,
|
|
2492
|
+
'cursor': cursor,
|
|
2493
|
+
'limit': 10
|
|
2494
|
+
}
|
|
2495
|
+
poll_url = f"https://backend-api.llumo.ai/api/v1/sdk/poll-insight-report"
|
|
2496
|
+
poll_response = requests.get(poll_url, params=poll_params)
|
|
2497
|
+
poll_response.raise_for_status()
|
|
2498
|
+
|
|
2499
|
+
data = poll_response.json()
|
|
2500
|
+
# Accessing nested data: pollResponse.data.debugLog
|
|
2501
|
+
debug_log = data.get('debugLog', {})
|
|
2502
|
+
results = debug_log.get('results', [])
|
|
2503
|
+
next_cursor = debug_log.get('nextCursor')
|
|
2504
|
+
|
|
2505
|
+
if results:
|
|
2506
|
+
insight_result.extend(results)
|
|
2507
|
+
|
|
2508
|
+
# Logic to handle cursor movement
|
|
2509
|
+
if next_cursor == cursor and not results:
|
|
2510
|
+
# print("Long poll returned empty (timeout). Continuing...")
|
|
2511
|
+
pass
|
|
2512
|
+
else:
|
|
2513
|
+
cursor = next_cursor
|
|
2514
|
+
|
|
2515
|
+
# Break condition for test (heuristic)
|
|
2516
|
+
if len(insight_result):
|
|
2517
|
+
break
|
|
2518
|
+
|
|
2519
|
+
except requests.exceptions.RequestException as e:
|
|
2520
|
+
error_msg = e.response.json() if e.response else str(e)
|
|
2521
|
+
# print(f"Error in poll request: {error_msg}")
|
|
2522
|
+
|
|
2523
|
+
if e.response is not None and e.response.status_code == 404:
|
|
2524
|
+
# print("Resource not ready yet...")
|
|
2525
|
+
pass
|
|
2526
|
+
else:
|
|
2527
|
+
# Fatal error, break the loop
|
|
2528
|
+
break
|
|
2529
|
+
|
|
2530
|
+
# Small delay to prevent tight loop (Equivalent to await sleep(1000))
|
|
2531
|
+
time.sleep(1)
|
|
2532
|
+
|
|
2533
|
+
# llumoInsight = formattedInsightResponse(llmResponse=insight_result)
|
|
2534
|
+
|
|
2535
|
+
return {"llumoInsight": insight_result}
|
|
2193
2536
|
|
|
2194
2537
|
|
|
2195
2538
|
class SafeDict(dict):
|
llumo/exceptions.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
|
|
1
2
|
class LlumoAIError(Exception):
|
|
2
3
|
"""Base class for all Llumo SDK-related errors."""
|
|
3
4
|
|
|
@@ -54,6 +55,14 @@ class LlumoAIError(Exception):
|
|
|
54
55
|
def providerError(details):
|
|
55
56
|
return LlumoAIError(details)
|
|
56
57
|
|
|
58
|
+
@staticmethod
|
|
59
|
+
def emptyLogList(details= "List of log object is empty. Ensure your logs have at least 1 log object."):
|
|
60
|
+
return LlumoAIError(details)
|
|
61
|
+
|
|
62
|
+
@staticmethod
|
|
63
|
+
def invalidUserAim(details= ""):
|
|
64
|
+
return LlumoAIError(details)
|
|
65
|
+
|
|
57
66
|
# @staticmethod
|
|
58
67
|
# def dateNotFound():
|
|
59
68
|
# return LlumoAIError("Trial end date or subscription end date not found for the given user.")
|
llumo/helpingFuntions.py
CHANGED
|
@@ -29,6 +29,9 @@ createInsightUrl="https://app.llumo.ai/api/external/generate-insight-from-eval-f
|
|
|
29
29
|
|
|
30
30
|
getCustomAnalyticsUrl="https://app.llumo.ai/api/workspace/get-all-analytics"
|
|
31
31
|
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
|
|
32
35
|
def getProcessID():
|
|
33
36
|
return f"{int(time.time() * 1000)}{uuid.uuid4()}"
|
|
34
37
|
|
|
@@ -739,6 +742,8 @@ def getCustomAnalytics(workspaceID):
|
|
|
739
742
|
return {}
|
|
740
743
|
|
|
741
744
|
|
|
745
|
+
def normalize_md(s: str) -> str:
|
|
746
|
+
return "\n".join(line.lstrip() for line in s.splitlines())
|
|
742
747
|
|
|
743
748
|
def postForDebugLogs(record: {},workspaceID):
|
|
744
749
|
url = "https://backend-api.llumo.ai/api/v1/get-debug-log-for-upload"
|
|
@@ -834,3 +839,104 @@ def addSelectedTools(run: dict) -> dict:
|
|
|
834
839
|
current_agent_step["metadata"]["toolSelected"].append(tool_name)
|
|
835
840
|
|
|
836
841
|
return run
|
|
842
|
+
|
|
843
|
+
def createMessageHistory(data, currentIndex = 0):
|
|
844
|
+
conversationHistory = []
|
|
845
|
+
for dataObj in data[:currentIndex]:
|
|
846
|
+
conversationHistory.append(dataObj)
|
|
847
|
+
return f"{conversationHistory}"
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
def addPromptAndInstructionInLogs(logData=None,promptTemplate= "",systemInstruction=""):
|
|
851
|
+
logDataWithPrompt = []
|
|
852
|
+
for data in logData:
|
|
853
|
+
if isinstance(data,str):
|
|
854
|
+
logDataWithPrompt.append(data + f"**promptTemplate**={promptTemplate}\n **systemInstruction**={systemInstruction}")
|
|
855
|
+
elif isinstance(data,dict):
|
|
856
|
+
data["promptTemplate"]= promptTemplate
|
|
857
|
+
data["systemInstruction"]= systemInstruction
|
|
858
|
+
logDataWithPrompt.append(data)
|
|
859
|
+
|
|
860
|
+
else:
|
|
861
|
+
return logData
|
|
862
|
+
return logDataWithPrompt
|
|
863
|
+
|
|
864
|
+
|
|
865
|
+
def dataPollingFuncForEval(headers,post_payload,poll_params,data_len:int):
|
|
866
|
+
postUrl = "https://backend-api.llumo.ai/api/v1/sdk/create-evaluation-Multiple"
|
|
867
|
+
pollUrl = "https://backend-api.llumo.ai/api/v1/sdk/poll"
|
|
868
|
+
|
|
869
|
+
# Create evaluation
|
|
870
|
+
requests.post(
|
|
871
|
+
url= postUrl,
|
|
872
|
+
json=post_payload,
|
|
873
|
+
headers=headers,
|
|
874
|
+
)
|
|
875
|
+
# print("payload", payload)
|
|
876
|
+
|
|
877
|
+
final_result_data = []
|
|
878
|
+
all_data_fetched = False
|
|
879
|
+
|
|
880
|
+
print("✅ Evaluation Started...")
|
|
881
|
+
while not all_data_fetched:
|
|
882
|
+
try:
|
|
883
|
+
response = requests.get(
|
|
884
|
+
pollUrl,
|
|
885
|
+
params=poll_params,
|
|
886
|
+
)
|
|
887
|
+
|
|
888
|
+
response_data = response.json()
|
|
889
|
+
result_data = response_data.get("debugLog", {})
|
|
890
|
+
# print("resultData", result_data)
|
|
891
|
+
|
|
892
|
+
results = result_data.get("results", [])
|
|
893
|
+
final_result_data.extend(results)
|
|
894
|
+
|
|
895
|
+
cursor = result_data.get("nextCursor")
|
|
896
|
+
|
|
897
|
+
if len(final_result_data) == data_len:
|
|
898
|
+
all_data_fetched = True
|
|
899
|
+
|
|
900
|
+
time.sleep(10)
|
|
901
|
+
|
|
902
|
+
except Exception as error:
|
|
903
|
+
print("error", error)
|
|
904
|
+
all_data_fetched = True
|
|
905
|
+
return final_result_data
|
|
906
|
+
|
|
907
|
+
def formattedInsightResponse(llmResponse):
|
|
908
|
+
try:
|
|
909
|
+
response = llmResponse
|
|
910
|
+
jsonResponse = response.json().get("response", {})
|
|
911
|
+
reasons = "\n- ".join(jsonResponse['reason'])
|
|
912
|
+
solutions = "\n- ".join(jsonResponse['solution'])
|
|
913
|
+
examples = "\n- ".join(jsonResponse['examples'])
|
|
914
|
+
|
|
915
|
+
formattedResponse = f"""
|
|
916
|
+
# 🔍 **{jsonResponse['insightTitle'].strip()}**
|
|
917
|
+
---
|
|
918
|
+
## 🧠 **Insight**
|
|
919
|
+
> {jsonResponse['insight'].strip()}
|
|
920
|
+
---
|
|
921
|
+
## 📝 **Description**
|
|
922
|
+
{jsonResponse['shortDescription'].strip()}
|
|
923
|
+
---
|
|
924
|
+
## 🔎 **Root Causes**
|
|
925
|
+
- {reasons.strip()}
|
|
926
|
+
---
|
|
927
|
+
## 🛠 **Solutions**
|
|
928
|
+
- {solutions.strip()}
|
|
929
|
+
---
|
|
930
|
+
## 📌 **Examples**
|
|
931
|
+
- {examples.strip()}
|
|
932
|
+
---
|
|
933
|
+
"""
|
|
934
|
+
formattedResponse = normalize_md(formattedResponse)
|
|
935
|
+
llumoInsight = formattedResponse
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
except Exception as e:
|
|
940
|
+
print("An error occurred. Please try again.")
|
|
941
|
+
llumoInsight = e
|
|
942
|
+
return llumoInsight
|
llumo/llumoLogger.py
CHANGED
|
@@ -2,9 +2,10 @@ import requests
|
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
class LlumoLogger:
|
|
5
|
-
def __init__(self, apiKey: str,
|
|
5
|
+
def __init__(self, apiKey: str, project: str):
|
|
6
6
|
self.apiKey = apiKey
|
|
7
|
-
self.playground = playground
|
|
7
|
+
# self.playground = playground
|
|
8
|
+
self.playground = project
|
|
8
9
|
self.workspaceID = None
|
|
9
10
|
self.playgroundID = None
|
|
10
11
|
self.userEmailID = None
|
llumo/llumoSessionContext.py
CHANGED
|
@@ -102,9 +102,10 @@ class LlumoSessionContext(LlumoClient):
|
|
|
102
102
|
"totalTokens": "",
|
|
103
103
|
"cost": "",
|
|
104
104
|
"modelsUsed": "gpt-4o",
|
|
105
|
+
"promptTemplate": promptTemplate,
|
|
106
|
+
"systemInstructions": systemInstructions
|
|
105
107
|
},
|
|
106
|
-
|
|
107
|
-
"systemInstructions":systemInstructions
|
|
108
|
+
|
|
108
109
|
}
|
|
109
110
|
|
|
110
111
|
self.threadLlumoRun = _ctxLlumoRun.set(llumoRun)
|
|
@@ -2,19 +2,19 @@ llumo/__init__.py,sha256=kkuppu7ZPiVZFdnYzJ9BM3syMbYHOSZLpwKwAvGHsnY,311
|
|
|
2
2
|
llumo/callback.py,sha256=XQbImLnd64_B2Iir-FCaJcL5Kmpm3RlXJH2zDRXk_yk,25135
|
|
3
3
|
llumo/callbacks-0.py,sha256=TEIOCWRvk2UYsTmBMBsnlgpqWvr-2y3a6d0w_e96NRM,8958
|
|
4
4
|
llumo/chains.py,sha256=6lCgLseh04RUgc6SahhmvQj82quay2Mi1j8gPUlx8Es,2923
|
|
5
|
-
llumo/client.py,sha256=
|
|
6
|
-
llumo/exceptions.py,sha256=
|
|
5
|
+
llumo/client.py,sha256=yuvh81zxkck-QvgqRSwTSGtLohkNBj28B_wsSf27NuQ,95904
|
|
6
|
+
llumo/exceptions.py,sha256=lUugHX1EorGse_5dU4vBRU82TNkN6VDaAq6JGOcfzaA,2381
|
|
7
7
|
llumo/execution.py,sha256=nWbJ7AvWuUPcOb6i-JzKRna_PvF-ewZTiK8skS-5n3w,1380
|
|
8
8
|
llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
|
|
9
9
|
llumo/google.py,sha256=6y9YnDFDRHv6-sQNT5LIsV9p31BCN0B9eow5KTRBWfM,2185
|
|
10
|
-
llumo/helpingFuntions.py,sha256=
|
|
11
|
-
llumo/llumoLogger.py,sha256=
|
|
12
|
-
llumo/llumoSessionContext.py,sha256=
|
|
10
|
+
llumo/helpingFuntions.py,sha256=4ompFeWq-mr67NkbiSUGvqFnQPtffIXUIYzGyihOqLs,33083
|
|
11
|
+
llumo/llumoLogger.py,sha256=syFerZgosq1xm2EkE-o6xBhEg7zE07X9ASao39ZEECQ,2256
|
|
12
|
+
llumo/llumoSessionContext.py,sha256=QaiQtB13dITTVINpzEAEVlO-fpW362XkgMjM1W4EHoo,14891
|
|
13
13
|
llumo/models.py,sha256=aVEZsOOoQx5LeNtwSyBxqvrINq0izH3QWu_YjsMPE6o,2910
|
|
14
14
|
llumo/openai.py,sha256=VstBzaORe8Tq0feUIIEszzcN1oq6TJfkPviaCr5d3Bw,8950
|
|
15
15
|
llumo/sockets.py,sha256=pfWz1zTEiwqJhdbSy3i3_Y4WlIdJ3cuac11wMePeBS0,6130
|
|
16
|
-
llumo-0.2.
|
|
17
|
-
llumo-0.2.
|
|
18
|
-
llumo-0.2.
|
|
19
|
-
llumo-0.2.
|
|
20
|
-
llumo-0.2.
|
|
16
|
+
llumo-0.2.43.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
|
|
17
|
+
llumo-0.2.43.dist-info/METADATA,sha256=z1TDEUY-imh53V2TxceOUO9_XXEmMuynH6RpfYEQPEc,1662
|
|
18
|
+
llumo-0.2.43.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
19
|
+
llumo-0.2.43.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
|
|
20
|
+
llumo-0.2.43.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|