llumo 0.2.42__py3-none-any.whl → 0.2.43__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llumo/client.py +364 -23
- llumo/exceptions.py +9 -0
- llumo/helpingFuntions.py +106 -0
- llumo/llumoLogger.py +3 -2
- {llumo-0.2.42.dist-info → llumo-0.2.43.dist-info}/METADATA +1 -1
- {llumo-0.2.42.dist-info → llumo-0.2.43.dist-info}/RECORD +9 -9
- {llumo-0.2.42.dist-info → llumo-0.2.43.dist-info}/WHEEL +0 -0
- {llumo-0.2.42.dist-info → llumo-0.2.43.dist-info}/licenses/LICENSE +0 -0
- {llumo-0.2.42.dist-info → llumo-0.2.43.dist-info}/top_level.txt +0 -0
llumo/client.py
CHANGED
|
@@ -8,10 +8,12 @@ import json
|
|
|
8
8
|
import uuid
|
|
9
9
|
import warnings
|
|
10
10
|
import os
|
|
11
|
-
|
|
11
|
+
from typing import List, Dict, Optional
|
|
12
12
|
import itertools
|
|
13
13
|
import pandas as pd
|
|
14
|
-
from typing import List, Dict
|
|
14
|
+
from typing import List, Dict, Any
|
|
15
|
+
|
|
16
|
+
|
|
15
17
|
from .models import AVAILABLEMODELS, getProviderFromModel, Provider
|
|
16
18
|
from .execution import ModelExecutor
|
|
17
19
|
from .exceptions import LlumoAIError
|
|
@@ -50,6 +52,7 @@ class LlumoClient:
|
|
|
50
52
|
self.evals = []
|
|
51
53
|
self.processMapping = {}
|
|
52
54
|
self.definationMapping = {}
|
|
55
|
+
self.ALL_USER_AIM = ['incorrectOutput', 'incorrectInput', 'hallucination', 'ragQuality', 'contextMismanagement', 'toolCallIssues', 'agentReasoning', 'stuckAgents', 'jsonErrors', 'highLatency', 'highCost', 'safetyBlocks', 'modelRouting', 'systemErrors', 'promptAdherence']
|
|
53
56
|
|
|
54
57
|
def validateApiKey(self, evalName="Input Bias"):
|
|
55
58
|
headers = {
|
|
@@ -780,7 +783,9 @@ class LlumoClient:
|
|
|
780
783
|
self,
|
|
781
784
|
data,
|
|
782
785
|
promptTemplate="",
|
|
783
|
-
systemInstructions = ""
|
|
786
|
+
systemInstructions = "",
|
|
787
|
+
multiTurnChat=False,
|
|
788
|
+
createMultipleLogs = True
|
|
784
789
|
|
|
785
790
|
):
|
|
786
791
|
if isinstance(data, dict):
|
|
@@ -788,7 +793,7 @@ class LlumoClient:
|
|
|
788
793
|
elif not isinstance(data, list):
|
|
789
794
|
raise ValueError("Data should be a dict or a list of dicts.")
|
|
790
795
|
|
|
791
|
-
dataframe = pd.DataFrame(data).astype(str)
|
|
796
|
+
dataframe = pd.DataFrame(data).astype(str).replace(to_replace="nan",value = "")
|
|
792
797
|
workspaceID = None
|
|
793
798
|
email = None
|
|
794
799
|
|
|
@@ -822,25 +827,45 @@ class LlumoClient:
|
|
|
822
827
|
# Extract required fields
|
|
823
828
|
query = row.get("query", "")
|
|
824
829
|
context = row.get("context", "")
|
|
830
|
+
|
|
825
831
|
tools = row.get("tools", "")
|
|
826
832
|
groundTruth = row.get("groundTruth", "")
|
|
827
|
-
|
|
833
|
+
|
|
834
|
+
if multiTurnChat==False:
|
|
835
|
+
# ---- SINGLE TURN (existing behavior) ----
|
|
836
|
+
messageHistory = row.get("messageHistory", "")
|
|
837
|
+
|
|
838
|
+
else:
|
|
839
|
+
# ---- MULTI TURN ----
|
|
840
|
+
multiTurnData = createMessageHistory(data, index)
|
|
841
|
+
|
|
842
|
+
if createMultipleLogs==True:
|
|
843
|
+
# each row will get history till that point
|
|
844
|
+
messageHistory = multiTurnData
|
|
845
|
+
else:
|
|
846
|
+
# only final API call should contain full history
|
|
847
|
+
if index == len(dataframe) - 1:
|
|
848
|
+
messageHistory = multiTurnData
|
|
849
|
+
else:
|
|
850
|
+
messageHistory = ""
|
|
851
|
+
|
|
852
|
+
|
|
853
|
+
|
|
828
854
|
intermediateSteps = row.get("intermediateSteps", "")
|
|
829
855
|
output = row.get("output", "")
|
|
830
856
|
|
|
831
|
-
# Initialize query and context
|
|
857
|
+
# # Initialize query and context
|
|
832
858
|
# query = ""
|
|
833
859
|
# context = ""
|
|
834
|
-
|
|
860
|
+
#
|
|
835
861
|
# # Process prompt template if provided
|
|
836
862
|
# if promptTemplate:
|
|
837
863
|
# # Extract template variables
|
|
838
864
|
# keys = re.findall(r"{{(.*?)}}", promptTemplate)
|
|
839
|
-
|
|
865
|
+
#
|
|
840
866
|
# if not all([key in dataframe.columns for key in keys]):
|
|
841
|
-
#
|
|
842
|
-
#
|
|
843
|
-
|
|
867
|
+
# raise LlumoAIError.InvalidPromptTemplate()
|
|
868
|
+
#
|
|
844
869
|
# # Populate template and separate query/context
|
|
845
870
|
# populated_template = promptTemplate
|
|
846
871
|
# for key in keys:
|
|
@@ -854,9 +879,9 @@ class LlumoClient:
|
|
|
854
879
|
# else:
|
|
855
880
|
# # Long value - add to context
|
|
856
881
|
# context += f" {key}: {value}, "
|
|
857
|
-
|
|
882
|
+
#
|
|
858
883
|
# query = populated_template.strip()
|
|
859
|
-
|
|
884
|
+
#
|
|
860
885
|
# # Add any remaining context from other fields
|
|
861
886
|
# if not context.strip():
|
|
862
887
|
# for key, value in row.items():
|
|
@@ -894,6 +919,7 @@ class LlumoClient:
|
|
|
894
919
|
|
|
895
920
|
|
|
896
921
|
}
|
|
922
|
+
|
|
897
923
|
currentTime = datetime(2025, 8, 2, 10, 20, 15, tzinfo=timezone.utc)
|
|
898
924
|
createdAt = currentTime.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
|
899
925
|
rowID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
|
|
@@ -926,7 +952,7 @@ class LlumoClient:
|
|
|
926
952
|
for i, batch in enumerate(allBatches, start=1):
|
|
927
953
|
|
|
928
954
|
try:
|
|
929
|
-
#
|
|
955
|
+
# print(batch)
|
|
930
956
|
response = postForDebugLogs(record=batch,workspaceID=workspaceID)
|
|
931
957
|
|
|
932
958
|
# failure case inside response
|
|
@@ -943,6 +969,9 @@ class LlumoClient:
|
|
|
943
969
|
|
|
944
970
|
print("Records Uploaded successfully. You may now review your logs at: https://app.llumo.ai/logs")
|
|
945
971
|
|
|
972
|
+
|
|
973
|
+
# Wait for results
|
|
974
|
+
|
|
946
975
|
# def evaluateMultiple(
|
|
947
976
|
# self,
|
|
948
977
|
# data,
|
|
@@ -2084,14 +2113,147 @@ class LlumoClient:
|
|
|
2084
2113
|
definationMapping=self.definationMapping,
|
|
2085
2114
|
)
|
|
2086
2115
|
|
|
2087
|
-
def
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
) -> List:
|
|
2116
|
+
# def get_evaluate_multiple(
|
|
2117
|
+
# self,
|
|
2118
|
+
# data,
|
|
2119
|
+
# evals
|
|
2120
|
+
# ) -> List:
|
|
2121
|
+
|
|
2122
|
+
# print("Evaluating multiple data with evals:", data, evals)
|
|
2123
|
+
|
|
2124
|
+
# dataID = uuid.uuid4().hex[:36]
|
|
2125
|
+
|
|
2126
|
+
# self.validateApiKey()
|
|
2092
2127
|
|
|
2128
|
+
# if not self.workspaceID:
|
|
2129
|
+
# raise LlumoAIError("Workspace ID not found after validation.")
|
|
2130
|
+
|
|
2131
|
+
# payload = {
|
|
2132
|
+
# "dataID": dataID,
|
|
2133
|
+
# "data": data,
|
|
2134
|
+
# "evals": evals,
|
|
2135
|
+
# "workspaceID": self.workspaceID,
|
|
2136
|
+
# "playgroundID": self.playgroundID,
|
|
2137
|
+
# }
|
|
2138
|
+
|
|
2139
|
+
# print("payload", payload)
|
|
2140
|
+
|
|
2141
|
+
# # Create evaluation
|
|
2142
|
+
# requests.post(
|
|
2143
|
+
# "https://backend-api.llumo.ai/api/v1/sdk/create-evaluation-Multiple",
|
|
2144
|
+
# json=payload,
|
|
2145
|
+
# headers={
|
|
2146
|
+
# "Content-Type": "application/json",
|
|
2147
|
+
# "Authorization": f"Bearer {self.apiKey}",
|
|
2148
|
+
# },
|
|
2149
|
+
# )
|
|
2150
|
+
|
|
2151
|
+
# final_result_data = []
|
|
2152
|
+
|
|
2153
|
+
# cursor = "0-0"
|
|
2154
|
+
# limit = 10
|
|
2155
|
+
# all_data_fetched = False
|
|
2156
|
+
|
|
2157
|
+
# while not all_data_fetched:
|
|
2158
|
+
# try:
|
|
2159
|
+
# response = requests.get(
|
|
2160
|
+
# "https://backend-api.llumo.ai/api/v1/sdk/poll",
|
|
2161
|
+
# params={
|
|
2162
|
+
# "cursor": cursor,
|
|
2163
|
+
# "dataID": dataID,
|
|
2164
|
+
# "limit": limit,
|
|
2165
|
+
# },
|
|
2166
|
+
# )
|
|
2167
|
+
|
|
2168
|
+
# response_data = response.json()
|
|
2169
|
+
# result_data = response_data.get("debugLog", {})
|
|
2170
|
+
# print("resultData", result_data)
|
|
2171
|
+
|
|
2172
|
+
# results = result_data.get("results", [])
|
|
2173
|
+
# final_result_data.extend(results)
|
|
2174
|
+
|
|
2175
|
+
# cursor = result_data.get("nextCursor")
|
|
2176
|
+
|
|
2177
|
+
# if len(final_result_data) == len(data):
|
|
2178
|
+
# all_data_fetched = True
|
|
2179
|
+
|
|
2180
|
+
# time.sleep(10)
|
|
2181
|
+
|
|
2182
|
+
# except Exception as error:
|
|
2183
|
+
# print("error", error)
|
|
2184
|
+
# all_data_fetched = True
|
|
2185
|
+
|
|
2186
|
+
# # Shape results
|
|
2187
|
+
# formatted_results = []
|
|
2188
|
+
|
|
2189
|
+
# for row in final_result_data:
|
|
2190
|
+
# score: Dict[str, float | None] = {}
|
|
2191
|
+
# reasoning: Dict[str, str] = {}
|
|
2192
|
+
|
|
2193
|
+
# for eval_name in evals:
|
|
2194
|
+
# details = row.get(eval_name)
|
|
2195
|
+
|
|
2196
|
+
# if isinstance(details, dict):
|
|
2197
|
+
# if isinstance(details.get("value"), (int, float)):
|
|
2198
|
+
# score[eval_name] = details.get("value")
|
|
2199
|
+
# else:
|
|
2200
|
+
# score[eval_name] = details.get("score")
|
|
2201
|
+
# reasoning[eval_name] = details.get("reasoning", "")
|
|
2202
|
+
|
|
2203
|
+
# elif "score" in row:
|
|
2204
|
+
# score[eval_name] = (
|
|
2205
|
+
# row["score"] if isinstance(row["score"], (int, float)) else None
|
|
2206
|
+
# )
|
|
2207
|
+
# reasoning[eval_name] = row.get("reasoning", "")
|
|
2208
|
+
# else:
|
|
2209
|
+
# score[eval_name] = None
|
|
2210
|
+
# reasoning[eval_name] = ""
|
|
2211
|
+
|
|
2212
|
+
# formatted_row = {
|
|
2213
|
+
# "context": row.get("context", ""),
|
|
2214
|
+
# "query": row.get("query", ""),
|
|
2215
|
+
# "output": row.get("output", ""),
|
|
2216
|
+
# "score": score,
|
|
2217
|
+
# "reasoning": reasoning,
|
|
2218
|
+
# }
|
|
2219
|
+
|
|
2220
|
+
# print(formatted_row)
|
|
2221
|
+
# formatted_results.append(formatted_row)
|
|
2222
|
+
|
|
2223
|
+
# return formatted_results
|
|
2224
|
+
|
|
2225
|
+
def getEvaluateMultiple(
|
|
2226
|
+
self,
|
|
2227
|
+
data,
|
|
2228
|
+
evals,
|
|
2229
|
+
promptTemplate="",
|
|
2230
|
+
systemInstructions="",
|
|
2231
|
+
multiTurnChat=False,
|
|
2232
|
+
createMultipleLogs=True
|
|
2233
|
+
):
|
|
2093
2234
|
# print("Evaluating multiple data with evals:", data, evals)
|
|
2235
|
+
try:
|
|
2236
|
+
self.validateApiKey()
|
|
2237
|
+
except Exception as e:
|
|
2238
|
+
print(f"Error during API key validation: {str(e)}")
|
|
2239
|
+
if hasattr(e, "response") and getattr(e, "response", None) is not None:
|
|
2240
|
+
print(f"Status code: {e.response.status_code}")
|
|
2241
|
+
print(f"Response content: {e.response.text[:500]}...")
|
|
2242
|
+
raise
|
|
2243
|
+
|
|
2244
|
+
userHits = checkUserHits(
|
|
2245
|
+
self.workspaceID,
|
|
2246
|
+
self.hasSubscribed,
|
|
2247
|
+
self.trialEndDate,
|
|
2248
|
+
self.subscriptionEndDate,
|
|
2249
|
+
self.hitsAvailable,
|
|
2250
|
+
len(data),
|
|
2251
|
+
)
|
|
2094
2252
|
|
|
2253
|
+
if not userHits["success"]:
|
|
2254
|
+
raise LlumoAIError.InsufficientCredits(userHits["message"])
|
|
2255
|
+
|
|
2256
|
+
print("✅ SDK integration successful!")
|
|
2095
2257
|
dataID = uuid.uuid4().hex[:36]
|
|
2096
2258
|
|
|
2097
2259
|
self.validateApiKey()
|
|
@@ -2099,6 +2261,38 @@ class LlumoClient:
|
|
|
2099
2261
|
if not self.workspaceID:
|
|
2100
2262
|
raise LlumoAIError("Workspace ID not found after validation.")
|
|
2101
2263
|
|
|
2264
|
+
if promptTemplate:
|
|
2265
|
+
for row in data:
|
|
2266
|
+
row["promptTemplate"] = promptTemplate
|
|
2267
|
+
if systemInstructions:
|
|
2268
|
+
for row in data:
|
|
2269
|
+
row["systemInstructions"] = systemInstructions
|
|
2270
|
+
|
|
2271
|
+
if multiTurnChat == True:
|
|
2272
|
+
if createMultipleLogs == True:
|
|
2273
|
+
dataWithMessageHistory = []
|
|
2274
|
+
for indx, row in enumerate(data):
|
|
2275
|
+
messageHistory = createMessageHistory(data, currentIndex=indx)
|
|
2276
|
+
rowCopy = row.copy()
|
|
2277
|
+
rowCopy["messageHistory"] = messageHistory
|
|
2278
|
+
dataWithMessageHistory.append(rowCopy)
|
|
2279
|
+
data = dataWithMessageHistory
|
|
2280
|
+
else:
|
|
2281
|
+
dataWithMessageHistory = []
|
|
2282
|
+
for indx, row in enumerate(data):
|
|
2283
|
+
if indx == len(data) - 1:
|
|
2284
|
+
messageHistory = createMessageHistory(data, currentIndex=indx)
|
|
2285
|
+
rowCopy = row.copy()
|
|
2286
|
+
rowCopy["messageHistory"] = messageHistory
|
|
2287
|
+
dataWithMessageHistory.append(rowCopy)
|
|
2288
|
+
else:
|
|
2289
|
+
row["messageHistory"] = ""
|
|
2290
|
+
dataWithMessageHistory.append(row)
|
|
2291
|
+
|
|
2292
|
+
data = dataWithMessageHistory
|
|
2293
|
+
|
|
2294
|
+
# print("DATA:")
|
|
2295
|
+
# print(data)
|
|
2102
2296
|
payload = {
|
|
2103
2297
|
"dataID": dataID,
|
|
2104
2298
|
"data": data,
|
|
@@ -2107,7 +2301,7 @@ class LlumoClient:
|
|
|
2107
2301
|
"playgroundID": self.playgroundID,
|
|
2108
2302
|
}
|
|
2109
2303
|
|
|
2110
|
-
#
|
|
2304
|
+
# print("payload", payload)
|
|
2111
2305
|
|
|
2112
2306
|
# Create evaluation
|
|
2113
2307
|
requests.post(
|
|
@@ -2125,6 +2319,7 @@ class LlumoClient:
|
|
|
2125
2319
|
limit = 10
|
|
2126
2320
|
all_data_fetched = False
|
|
2127
2321
|
|
|
2322
|
+
print("✅ Evaluation Started...")
|
|
2128
2323
|
while not all_data_fetched:
|
|
2129
2324
|
try:
|
|
2130
2325
|
response = requests.get(
|
|
@@ -2138,7 +2333,7 @@ class LlumoClient:
|
|
|
2138
2333
|
|
|
2139
2334
|
response_data = response.json()
|
|
2140
2335
|
result_data = response_data.get("debugLog", {})
|
|
2141
|
-
print("resultData", result_data)
|
|
2336
|
+
# print("resultData", result_data)
|
|
2142
2337
|
|
|
2143
2338
|
results = result_data.get("results", [])
|
|
2144
2339
|
final_result_data.extend(results)
|
|
@@ -2181,8 +2376,8 @@ class LlumoClient:
|
|
|
2181
2376
|
reasoning[eval_name] = ""
|
|
2182
2377
|
|
|
2183
2378
|
formatted_row = {
|
|
2184
|
-
"context": row.get("context", ""),
|
|
2185
2379
|
"query": row.get("query", ""),
|
|
2380
|
+
"context": row.get("context", ""),
|
|
2186
2381
|
"output": row.get("output", ""),
|
|
2187
2382
|
"score": score,
|
|
2188
2383
|
"reasoning": reasoning,
|
|
@@ -2191,7 +2386,153 @@ class LlumoClient:
|
|
|
2191
2386
|
# print(formatted_row)
|
|
2192
2387
|
formatted_results.append(formatted_row)
|
|
2193
2388
|
|
|
2194
|
-
return formatted_results
|
|
2389
|
+
return {"llumoEval": formatted_results}
|
|
2390
|
+
|
|
2391
|
+
|
|
2392
|
+
def getInsights(self,logs:List,userAim:List[str],promptTemplate:str = ""
|
|
2393
|
+
,systemInstructions:str="",multiTurnChat=False,createMultipleLogs=True):
|
|
2394
|
+
|
|
2395
|
+
try:
|
|
2396
|
+
self.validateApiKey()
|
|
2397
|
+
except Exception as e:
|
|
2398
|
+
print(f"Error during API key validation: {str(e)}")
|
|
2399
|
+
if hasattr(e, "response") and getattr(e, "response", None) is not None:
|
|
2400
|
+
print(f"Status code: {e.response.status_code}")
|
|
2401
|
+
print(f"Response content: {e.response.text[:500]}...")
|
|
2402
|
+
raise
|
|
2403
|
+
|
|
2404
|
+
userHits = checkUserHits(
|
|
2405
|
+
self.workspaceID,
|
|
2406
|
+
self.hasSubscribed,
|
|
2407
|
+
self.trialEndDate,
|
|
2408
|
+
self.subscriptionEndDate,
|
|
2409
|
+
self.hitsAvailable,
|
|
2410
|
+
1,
|
|
2411
|
+
)
|
|
2412
|
+
|
|
2413
|
+
if not userHits["success"]:
|
|
2414
|
+
raise LlumoAIError.InsufficientCredits(userHits["message"])
|
|
2415
|
+
|
|
2416
|
+
if len(logs)==0 :
|
|
2417
|
+
raise LlumoAIError.emptyLogList()
|
|
2418
|
+
|
|
2419
|
+
if not isinstance(userAim, list):
|
|
2420
|
+
raise TypeError(f"userAim must be list, got {type(userAim).__name__}")
|
|
2421
|
+
|
|
2422
|
+
if any(aim not in self.ALL_USER_AIM for aim in userAim):
|
|
2423
|
+
errorMessage = f"Please pass a valid user aim. Only acceptable user aims are->{self.ALL_USER_AIM}"
|
|
2424
|
+
raise LlumoAIError.invalidUserAim(details=errorMessage)
|
|
2425
|
+
|
|
2426
|
+
|
|
2427
|
+
|
|
2428
|
+
if multiTurnChat == True:
|
|
2429
|
+
if createMultipleLogs == True:
|
|
2430
|
+
dataWithMessageHistory = []
|
|
2431
|
+
for indx, row in enumerate(logs):
|
|
2432
|
+
messageHistory = createMessageHistory(logs, currentIndex=indx)
|
|
2433
|
+
rowCopy = row.copy()
|
|
2434
|
+
rowCopy["messageHistory"] = messageHistory
|
|
2435
|
+
dataWithMessageHistory.append(rowCopy)
|
|
2436
|
+
logs= dataWithMessageHistory
|
|
2437
|
+
else:
|
|
2438
|
+
dataWithMessageHistory = []
|
|
2439
|
+
for indx, row in enumerate(logs):
|
|
2440
|
+
if indx == len(logs) - 1:
|
|
2441
|
+
messageHistory = createMessageHistory(logs, currentIndex=indx)
|
|
2442
|
+
rowCopy = row.copy()
|
|
2443
|
+
rowCopy["messageHistory"] = messageHistory
|
|
2444
|
+
dataWithMessageHistory.append(rowCopy)
|
|
2445
|
+
else:
|
|
2446
|
+
row["messageHistory"] = ""
|
|
2447
|
+
dataWithMessageHistory.append(row)
|
|
2448
|
+
logs = dataWithMessageHistory
|
|
2449
|
+
|
|
2450
|
+
if (promptTemplate!="") or systemInstructions!="":
|
|
2451
|
+
logs = addPromptAndInstructionInLogs(logData = logs ,promptTemplate=promptTemplate,systemInstruction=systemInstructions)
|
|
2452
|
+
|
|
2453
|
+
# print("[LOGS: ]")
|
|
2454
|
+
# print(logs)
|
|
2455
|
+
|
|
2456
|
+
# 1. Create Report
|
|
2457
|
+
print("✅ Generating Insights Now....")
|
|
2458
|
+
dataID = uuid.uuid4().hex[:36]
|
|
2459
|
+
try:
|
|
2460
|
+
payload = {
|
|
2461
|
+
"data":logs,
|
|
2462
|
+
"userAim":userAim,
|
|
2463
|
+
"dataID":dataID
|
|
2464
|
+
}
|
|
2465
|
+
create_url = "https://backend-api.llumo.ai/api/v1/sdk/create-insight-report"
|
|
2466
|
+
response = requests.post(create_url, json=payload)
|
|
2467
|
+
|
|
2468
|
+
# Check if request was successful
|
|
2469
|
+
response.raise_for_status()
|
|
2470
|
+
|
|
2471
|
+
# print(f"Create Response: {response.json()}")
|
|
2472
|
+
except requests.exceptions.RequestException as e:
|
|
2473
|
+
# Check for response data in the error
|
|
2474
|
+
error_data = e.response.json() if e.response else str(e)
|
|
2475
|
+
print(f"Error in create request: {error_data}")
|
|
2476
|
+
return
|
|
2477
|
+
|
|
2478
|
+
# 2. Poll for Results
|
|
2479
|
+
# print("\n--- Starting Polling ---")
|
|
2480
|
+
cursor = '0-0'
|
|
2481
|
+
is_complete = False
|
|
2482
|
+
max_polls = 20
|
|
2483
|
+
poll_count = 0
|
|
2484
|
+
|
|
2485
|
+
insight_result = []
|
|
2486
|
+
while not is_complete and poll_count < max_polls:
|
|
2487
|
+
poll_count += 1
|
|
2488
|
+
|
|
2489
|
+
try:
|
|
2490
|
+
poll_params = {
|
|
2491
|
+
'dataID': dataID,
|
|
2492
|
+
'cursor': cursor,
|
|
2493
|
+
'limit': 10
|
|
2494
|
+
}
|
|
2495
|
+
poll_url = f"https://backend-api.llumo.ai/api/v1/sdk/poll-insight-report"
|
|
2496
|
+
poll_response = requests.get(poll_url, params=poll_params)
|
|
2497
|
+
poll_response.raise_for_status()
|
|
2498
|
+
|
|
2499
|
+
data = poll_response.json()
|
|
2500
|
+
# Accessing nested data: pollResponse.data.debugLog
|
|
2501
|
+
debug_log = data.get('debugLog', {})
|
|
2502
|
+
results = debug_log.get('results', [])
|
|
2503
|
+
next_cursor = debug_log.get('nextCursor')
|
|
2504
|
+
|
|
2505
|
+
if results:
|
|
2506
|
+
insight_result.extend(results)
|
|
2507
|
+
|
|
2508
|
+
# Logic to handle cursor movement
|
|
2509
|
+
if next_cursor == cursor and not results:
|
|
2510
|
+
# print("Long poll returned empty (timeout). Continuing...")
|
|
2511
|
+
pass
|
|
2512
|
+
else:
|
|
2513
|
+
cursor = next_cursor
|
|
2514
|
+
|
|
2515
|
+
# Break condition for test (heuristic)
|
|
2516
|
+
if len(insight_result):
|
|
2517
|
+
break
|
|
2518
|
+
|
|
2519
|
+
except requests.exceptions.RequestException as e:
|
|
2520
|
+
error_msg = e.response.json() if e.response else str(e)
|
|
2521
|
+
# print(f"Error in poll request: {error_msg}")
|
|
2522
|
+
|
|
2523
|
+
if e.response is not None and e.response.status_code == 404:
|
|
2524
|
+
# print("Resource not ready yet...")
|
|
2525
|
+
pass
|
|
2526
|
+
else:
|
|
2527
|
+
# Fatal error, break the loop
|
|
2528
|
+
break
|
|
2529
|
+
|
|
2530
|
+
# Small delay to prevent tight loop (Equivalent to await sleep(1000))
|
|
2531
|
+
time.sleep(1)
|
|
2532
|
+
|
|
2533
|
+
# llumoInsight = formattedInsightResponse(llmResponse=insight_result)
|
|
2534
|
+
|
|
2535
|
+
return {"llumoInsight": insight_result}
|
|
2195
2536
|
|
|
2196
2537
|
|
|
2197
2538
|
class SafeDict(dict):
|
llumo/exceptions.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
|
|
1
2
|
class LlumoAIError(Exception):
|
|
2
3
|
"""Base class for all Llumo SDK-related errors."""
|
|
3
4
|
|
|
@@ -54,6 +55,14 @@ class LlumoAIError(Exception):
|
|
|
54
55
|
def providerError(details):
|
|
55
56
|
return LlumoAIError(details)
|
|
56
57
|
|
|
58
|
+
@staticmethod
|
|
59
|
+
def emptyLogList(details= "List of log object is empty. Ensure your logs have at least 1 log object."):
|
|
60
|
+
return LlumoAIError(details)
|
|
61
|
+
|
|
62
|
+
@staticmethod
|
|
63
|
+
def invalidUserAim(details= ""):
|
|
64
|
+
return LlumoAIError(details)
|
|
65
|
+
|
|
57
66
|
# @staticmethod
|
|
58
67
|
# def dateNotFound():
|
|
59
68
|
# return LlumoAIError("Trial end date or subscription end date not found for the given user.")
|
llumo/helpingFuntions.py
CHANGED
|
@@ -29,6 +29,9 @@ createInsightUrl="https://app.llumo.ai/api/external/generate-insight-from-eval-f
|
|
|
29
29
|
|
|
30
30
|
getCustomAnalyticsUrl="https://app.llumo.ai/api/workspace/get-all-analytics"
|
|
31
31
|
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
|
|
32
35
|
def getProcessID():
|
|
33
36
|
return f"{int(time.time() * 1000)}{uuid.uuid4()}"
|
|
34
37
|
|
|
@@ -739,6 +742,8 @@ def getCustomAnalytics(workspaceID):
|
|
|
739
742
|
return {}
|
|
740
743
|
|
|
741
744
|
|
|
745
|
+
def normalize_md(s: str) -> str:
|
|
746
|
+
return "\n".join(line.lstrip() for line in s.splitlines())
|
|
742
747
|
|
|
743
748
|
def postForDebugLogs(record: {},workspaceID):
|
|
744
749
|
url = "https://backend-api.llumo.ai/api/v1/get-debug-log-for-upload"
|
|
@@ -834,3 +839,104 @@ def addSelectedTools(run: dict) -> dict:
|
|
|
834
839
|
current_agent_step["metadata"]["toolSelected"].append(tool_name)
|
|
835
840
|
|
|
836
841
|
return run
|
|
842
|
+
|
|
843
|
+
def createMessageHistory(data, currentIndex = 0):
|
|
844
|
+
conversationHistory = []
|
|
845
|
+
for dataObj in data[:currentIndex]:
|
|
846
|
+
conversationHistory.append(dataObj)
|
|
847
|
+
return f"{conversationHistory}"
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
def addPromptAndInstructionInLogs(logData=None,promptTemplate= "",systemInstruction=""):
|
|
851
|
+
logDataWithPrompt = []
|
|
852
|
+
for data in logData:
|
|
853
|
+
if isinstance(data,str):
|
|
854
|
+
logDataWithPrompt.append(data + f"**promptTemplate**={promptTemplate}\n **systemInstruction**={systemInstruction}")
|
|
855
|
+
elif isinstance(data,dict):
|
|
856
|
+
data["promptTemplate"]= promptTemplate
|
|
857
|
+
data["systemInstruction"]= systemInstruction
|
|
858
|
+
logDataWithPrompt.append(data)
|
|
859
|
+
|
|
860
|
+
else:
|
|
861
|
+
return logData
|
|
862
|
+
return logDataWithPrompt
|
|
863
|
+
|
|
864
|
+
|
|
865
|
+
def dataPollingFuncForEval(headers,post_payload,poll_params,data_len:int):
|
|
866
|
+
postUrl = "https://backend-api.llumo.ai/api/v1/sdk/create-evaluation-Multiple"
|
|
867
|
+
pollUrl = "https://backend-api.llumo.ai/api/v1/sdk/poll"
|
|
868
|
+
|
|
869
|
+
# Create evaluation
|
|
870
|
+
requests.post(
|
|
871
|
+
url= postUrl,
|
|
872
|
+
json=post_payload,
|
|
873
|
+
headers=headers,
|
|
874
|
+
)
|
|
875
|
+
# print("payload", payload)
|
|
876
|
+
|
|
877
|
+
final_result_data = []
|
|
878
|
+
all_data_fetched = False
|
|
879
|
+
|
|
880
|
+
print("✅ Evaluation Started...")
|
|
881
|
+
while not all_data_fetched:
|
|
882
|
+
try:
|
|
883
|
+
response = requests.get(
|
|
884
|
+
pollUrl,
|
|
885
|
+
params=poll_params,
|
|
886
|
+
)
|
|
887
|
+
|
|
888
|
+
response_data = response.json()
|
|
889
|
+
result_data = response_data.get("debugLog", {})
|
|
890
|
+
# print("resultData", result_data)
|
|
891
|
+
|
|
892
|
+
results = result_data.get("results", [])
|
|
893
|
+
final_result_data.extend(results)
|
|
894
|
+
|
|
895
|
+
cursor = result_data.get("nextCursor")
|
|
896
|
+
|
|
897
|
+
if len(final_result_data) == data_len:
|
|
898
|
+
all_data_fetched = True
|
|
899
|
+
|
|
900
|
+
time.sleep(10)
|
|
901
|
+
|
|
902
|
+
except Exception as error:
|
|
903
|
+
print("error", error)
|
|
904
|
+
all_data_fetched = True
|
|
905
|
+
return final_result_data
|
|
906
|
+
|
|
907
|
+
def formattedInsightResponse(llmResponse):
|
|
908
|
+
try:
|
|
909
|
+
response = llmResponse
|
|
910
|
+
jsonResponse = response.json().get("response", {})
|
|
911
|
+
reasons = "\n- ".join(jsonResponse['reason'])
|
|
912
|
+
solutions = "\n- ".join(jsonResponse['solution'])
|
|
913
|
+
examples = "\n- ".join(jsonResponse['examples'])
|
|
914
|
+
|
|
915
|
+
formattedResponse = f"""
|
|
916
|
+
# 🔍 **{jsonResponse['insightTitle'].strip()}**
|
|
917
|
+
---
|
|
918
|
+
## 🧠 **Insight**
|
|
919
|
+
> {jsonResponse['insight'].strip()}
|
|
920
|
+
---
|
|
921
|
+
## 📝 **Description**
|
|
922
|
+
{jsonResponse['shortDescription'].strip()}
|
|
923
|
+
---
|
|
924
|
+
## 🔎 **Root Causes**
|
|
925
|
+
- {reasons.strip()}
|
|
926
|
+
---
|
|
927
|
+
## 🛠 **Solutions**
|
|
928
|
+
- {solutions.strip()}
|
|
929
|
+
---
|
|
930
|
+
## 📌 **Examples**
|
|
931
|
+
- {examples.strip()}
|
|
932
|
+
---
|
|
933
|
+
"""
|
|
934
|
+
formattedResponse = normalize_md(formattedResponse)
|
|
935
|
+
llumoInsight = formattedResponse
|
|
936
|
+
|
|
937
|
+
|
|
938
|
+
|
|
939
|
+
except Exception as e:
|
|
940
|
+
print("An error occurred. Please try again.")
|
|
941
|
+
llumoInsight = e
|
|
942
|
+
return llumoInsight
|
llumo/llumoLogger.py
CHANGED
|
@@ -2,9 +2,10 @@ import requests
|
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
class LlumoLogger:
|
|
5
|
-
def __init__(self, apiKey: str,
|
|
5
|
+
def __init__(self, apiKey: str, project: str):
|
|
6
6
|
self.apiKey = apiKey
|
|
7
|
-
self.playground = playground
|
|
7
|
+
# self.playground = playground
|
|
8
|
+
self.playground = project
|
|
8
9
|
self.workspaceID = None
|
|
9
10
|
self.playgroundID = None
|
|
10
11
|
self.userEmailID = None
|
|
@@ -2,19 +2,19 @@ llumo/__init__.py,sha256=kkuppu7ZPiVZFdnYzJ9BM3syMbYHOSZLpwKwAvGHsnY,311
|
|
|
2
2
|
llumo/callback.py,sha256=XQbImLnd64_B2Iir-FCaJcL5Kmpm3RlXJH2zDRXk_yk,25135
|
|
3
3
|
llumo/callbacks-0.py,sha256=TEIOCWRvk2UYsTmBMBsnlgpqWvr-2y3a6d0w_e96NRM,8958
|
|
4
4
|
llumo/chains.py,sha256=6lCgLseh04RUgc6SahhmvQj82quay2Mi1j8gPUlx8Es,2923
|
|
5
|
-
llumo/client.py,sha256=
|
|
6
|
-
llumo/exceptions.py,sha256=
|
|
5
|
+
llumo/client.py,sha256=yuvh81zxkck-QvgqRSwTSGtLohkNBj28B_wsSf27NuQ,95904
|
|
6
|
+
llumo/exceptions.py,sha256=lUugHX1EorGse_5dU4vBRU82TNkN6VDaAq6JGOcfzaA,2381
|
|
7
7
|
llumo/execution.py,sha256=nWbJ7AvWuUPcOb6i-JzKRna_PvF-ewZTiK8skS-5n3w,1380
|
|
8
8
|
llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
|
|
9
9
|
llumo/google.py,sha256=6y9YnDFDRHv6-sQNT5LIsV9p31BCN0B9eow5KTRBWfM,2185
|
|
10
|
-
llumo/helpingFuntions.py,sha256=
|
|
11
|
-
llumo/llumoLogger.py,sha256=
|
|
10
|
+
llumo/helpingFuntions.py,sha256=4ompFeWq-mr67NkbiSUGvqFnQPtffIXUIYzGyihOqLs,33083
|
|
11
|
+
llumo/llumoLogger.py,sha256=syFerZgosq1xm2EkE-o6xBhEg7zE07X9ASao39ZEECQ,2256
|
|
12
12
|
llumo/llumoSessionContext.py,sha256=QaiQtB13dITTVINpzEAEVlO-fpW362XkgMjM1W4EHoo,14891
|
|
13
13
|
llumo/models.py,sha256=aVEZsOOoQx5LeNtwSyBxqvrINq0izH3QWu_YjsMPE6o,2910
|
|
14
14
|
llumo/openai.py,sha256=VstBzaORe8Tq0feUIIEszzcN1oq6TJfkPviaCr5d3Bw,8950
|
|
15
15
|
llumo/sockets.py,sha256=pfWz1zTEiwqJhdbSy3i3_Y4WlIdJ3cuac11wMePeBS0,6130
|
|
16
|
-
llumo-0.2.
|
|
17
|
-
llumo-0.2.
|
|
18
|
-
llumo-0.2.
|
|
19
|
-
llumo-0.2.
|
|
20
|
-
llumo-0.2.
|
|
16
|
+
llumo-0.2.43.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
|
|
17
|
+
llumo-0.2.43.dist-info/METADATA,sha256=z1TDEUY-imh53V2TxceOUO9_XXEmMuynH6RpfYEQPEc,1662
|
|
18
|
+
llumo-0.2.43.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
19
|
+
llumo-0.2.43.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
|
|
20
|
+
llumo-0.2.43.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|