llumo 0.2.14b4__py3-none-any.whl → 0.2.14b6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llumo/client.py +221 -105
- llumo/helpingFuntions.py +6 -6
- llumo/sockets.py +20 -9
- {llumo-0.2.14b4.dist-info → llumo-0.2.14b6.dist-info}/METADATA +1 -1
- llumo-0.2.14b6.dist-info/RECORD +13 -0
- llumo-0.2.14b4.dist-info/RECORD +0 -13
- {llumo-0.2.14b4.dist-info → llumo-0.2.14b6.dist-info}/WHEEL +0 -0
- {llumo-0.2.14b4.dist-info → llumo-0.2.14b6.dist-info}/licenses/LICENSE +0 -0
- {llumo-0.2.14b4.dist-info → llumo-0.2.14b6.dist-info}/top_level.txt +0 -0
llumo/client.py
CHANGED
@@ -19,8 +19,12 @@ from .functionCalling import LlumoAgentExecutor
|
|
19
19
|
import threading
|
20
20
|
from tqdm import tqdm
|
21
21
|
|
22
|
-
postUrl =
|
23
|
-
|
22
|
+
postUrl = (
|
23
|
+
"https://red-skull-service-392377961931.us-central1.run.app/api/process-playground"
|
24
|
+
)
|
25
|
+
fetchUrl = (
|
26
|
+
"https://red-skull-service-392377961931.us-central1.run.app/api/get-cells-data"
|
27
|
+
)
|
24
28
|
validateUrl = "https://app.llumo.ai/api/workspace-details"
|
25
29
|
socketUrl = "https://red-skull-service-392377961931.us-central1.run.app/"
|
26
30
|
|
@@ -42,9 +46,6 @@ class LlumoClient:
|
|
42
46
|
|
43
47
|
try:
|
44
48
|
response = requests.post(url=validateUrl, json=reqBody, headers=headers)
|
45
|
-
|
46
|
-
|
47
|
-
|
48
49
|
|
49
50
|
except requests.exceptions.RequestException as e:
|
50
51
|
print(f"Request exception: {str(e)}")
|
@@ -77,16 +78,20 @@ class LlumoClient:
|
|
77
78
|
raise LlumoAIError.InvalidApiResponse()
|
78
79
|
|
79
80
|
try:
|
80
|
-
self.hitsAvailable = data[
|
81
|
+
self.hitsAvailable = data["data"]["data"].get("remainingHits", 0)
|
81
82
|
self.workspaceID = data["data"]["data"].get("workspaceID")
|
82
83
|
self.evalDefinition = data["data"]["data"]["analyticsMapping"]
|
83
84
|
self.socketToken = data["data"]["data"].get("token")
|
84
85
|
self.hasSubscribed = data["data"]["data"].get("hasSubscribed", False)
|
85
86
|
self.trialEndDate = data["data"]["data"].get("trialEndDate", None)
|
86
|
-
self.subscriptionEndDate = data["data"]["data"].get(
|
87
|
+
self.subscriptionEndDate = data["data"]["data"].get(
|
88
|
+
"subscriptionEndDate", None
|
89
|
+
)
|
87
90
|
self.email = data["data"]["data"].get("email", None)
|
88
|
-
|
89
|
-
self.definationMapping[evalName] = data["data"]["data"]["analyticsMapping"][
|
91
|
+
|
92
|
+
self.definationMapping[evalName] = data["data"]["data"]["analyticsMapping"][
|
93
|
+
evalName
|
94
|
+
]
|
90
95
|
except Exception as e:
|
91
96
|
# print(f"Error extracting data from response: {str(e)}")
|
92
97
|
raise LlumoAIError.UnexpectedError(detail=str(e))
|
@@ -160,14 +165,16 @@ class LlumoClient:
|
|
160
165
|
prompt_template="",
|
161
166
|
outputColName="output",
|
162
167
|
createExperiment: bool = False,
|
163
|
-
_tocheck
|
168
|
+
_tocheck=True,
|
164
169
|
):
|
165
|
-
|
170
|
+
|
166
171
|
# converting it into a pandas dataframe object
|
167
172
|
dataframe = pd.DataFrame(data)
|
168
173
|
|
169
174
|
# check for dependencies for the selected eval metric
|
170
|
-
metricDependencies = checkDependency(
|
175
|
+
metricDependencies = checkDependency(
|
176
|
+
eval, columns=list(dataframe.columns), tocheck=_tocheck
|
177
|
+
)
|
171
178
|
if metricDependencies["status"] == False:
|
172
179
|
raise LlumoAIError.dependencyError(metricDependencies["message"])
|
173
180
|
|
@@ -323,7 +330,7 @@ class LlumoClient:
|
|
323
330
|
|
324
331
|
for cnt, batch in enumerate(self.allBatches):
|
325
332
|
try:
|
326
|
-
|
333
|
+
|
327
334
|
self.postBatch(batch=batch, workspaceID=workspaceID)
|
328
335
|
print("Betch Posted with item len: ", len(batch))
|
329
336
|
except Exception as e:
|
@@ -372,7 +379,14 @@ class LlumoClient:
|
|
372
379
|
pd.set_option("future.no_silent_downcasting", True)
|
373
380
|
df = dataframe.fillna("Some error occured").astype(object)
|
374
381
|
|
375
|
-
if createPlayground(
|
382
|
+
if createPlayground(
|
383
|
+
email,
|
384
|
+
workspaceID,
|
385
|
+
df,
|
386
|
+
promptText=prompt_template,
|
387
|
+
definationMapping=self.definationMapping,
|
388
|
+
outputColName=outputColName,
|
389
|
+
):
|
376
390
|
print(
|
377
391
|
"Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results.Please rerun the experiment to see the results on playground."
|
378
392
|
)
|
@@ -392,7 +406,9 @@ class LlumoClient:
|
|
392
406
|
time.sleep(0.1)
|
393
407
|
waited_secs += 0.1
|
394
408
|
if waited_secs >= max_wait_secs:
|
395
|
-
raise RuntimeError(
|
409
|
+
raise RuntimeError(
|
410
|
+
"Timeout waiting for server 'connection-established' event."
|
411
|
+
)
|
396
412
|
|
397
413
|
try:
|
398
414
|
self.validateApiKey()
|
@@ -403,8 +419,14 @@ class LlumoClient:
|
|
403
419
|
print(f"Response content: {e.response.text[:500]}...")
|
404
420
|
raise
|
405
421
|
|
406
|
-
userHits = checkUserHits(
|
407
|
-
|
422
|
+
userHits = checkUserHits(
|
423
|
+
self.workspaceID,
|
424
|
+
self.hasSubscribed,
|
425
|
+
self.trialEndDate,
|
426
|
+
self.subscriptionEndDate,
|
427
|
+
self.hitsAvailable,
|
428
|
+
len(dataframe),
|
429
|
+
)
|
408
430
|
|
409
431
|
if not userHits["success"]:
|
410
432
|
raise LlumoAIError.InsufficientCredits(userHits["message"])
|
@@ -425,14 +447,22 @@ class LlumoClient:
|
|
425
447
|
if not all([ky in dataframe.columns for ky in keys]):
|
426
448
|
raise LlumoAIError.InvalidPromptTemplate()
|
427
449
|
|
428
|
-
activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
|
450
|
+
activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
|
451
|
+
"-", ""
|
452
|
+
)
|
429
453
|
rowID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
|
430
454
|
columnID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
|
431
455
|
|
432
|
-
compressed_prompt_id =
|
433
|
-
|
456
|
+
compressed_prompt_id = (
|
457
|
+
f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
|
458
|
+
)
|
459
|
+
compressed_prompt_output_id = (
|
460
|
+
f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
|
461
|
+
)
|
434
462
|
cost_id = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
|
435
|
-
cost_saving_id = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
|
463
|
+
cost_saving_id = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
|
464
|
+
"-", ""
|
465
|
+
)
|
436
466
|
|
437
467
|
rowDataDict = {}
|
438
468
|
for col in dataframe.columns:
|
@@ -452,7 +482,7 @@ class LlumoClient:
|
|
452
482
|
"compressed_prompt": compressed_prompt_id,
|
453
483
|
"compressed_prompt_output": compressed_prompt_output_id,
|
454
484
|
"cost": cost_id,
|
455
|
-
"cost_saving": cost_saving_id
|
485
|
+
"cost_saving": cost_saving_id,
|
456
486
|
},
|
457
487
|
"processData": {
|
458
488
|
"rowData": rowDataDict,
|
@@ -465,12 +495,12 @@ class LlumoClient:
|
|
465
495
|
"compressed_prompt": compressed_prompt_id,
|
466
496
|
"compressed_prompt_output": compressed_prompt_output_id,
|
467
497
|
"cost": cost_id,
|
468
|
-
"cost_saving": cost_saving_id
|
469
|
-
}
|
498
|
+
"cost_saving": cost_saving_id,
|
499
|
+
},
|
470
500
|
},
|
471
501
|
"workspaceID": workspaceID,
|
472
502
|
"email": email,
|
473
|
-
"playgroundID": activePlayground
|
503
|
+
"playgroundID": activePlayground,
|
474
504
|
}
|
475
505
|
|
476
506
|
rowIdMapping[rowID] = index
|
@@ -495,7 +525,12 @@ class LlumoClient:
|
|
495
525
|
|
496
526
|
self.AllProcessMapping()
|
497
527
|
timeout = max(60, min(600, total_items * 10))
|
498
|
-
self.socket.listenForResults(
|
528
|
+
self.socket.listenForResults(
|
529
|
+
min_wait=20,
|
530
|
+
max_wait=timeout,
|
531
|
+
inactivity_timeout=30,
|
532
|
+
expected_results=None,
|
533
|
+
)
|
499
534
|
|
500
535
|
results = self.socket.getReceivedData()
|
501
536
|
# results = self.finalResp(eval_results)
|
@@ -514,7 +549,7 @@ class LlumoClient:
|
|
514
549
|
for records in results:
|
515
550
|
for compound_key, value in records.items():
|
516
551
|
# for compound_key, value in item['data'].items():
|
517
|
-
rowID = compound_key.split(
|
552
|
+
rowID = compound_key.split("-")[0]
|
518
553
|
# looking for the index of each rowID , in the original dataframe
|
519
554
|
if rowID in rowIdMapping:
|
520
555
|
index = rowIdMapping[rowID]
|
@@ -534,18 +569,18 @@ class LlumoClient:
|
|
534
569
|
return dataframe
|
535
570
|
|
536
571
|
def evaluateMultiple(
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
|
542
|
-
|
543
|
-
|
572
|
+
self,
|
573
|
+
data,
|
574
|
+
evals: list, # list of eval metric names
|
575
|
+
prompt_template="",
|
576
|
+
outputColName="output",
|
577
|
+
createExperiment: bool = False,
|
578
|
+
_tocheck=True,
|
544
579
|
):
|
545
580
|
dataframe = pd.DataFrame(data)
|
546
581
|
workspaceID = None
|
547
582
|
email = None
|
548
|
-
socketID = self.socket.connect(timeout=
|
583
|
+
socketID = self.socket.connect(timeout=250)
|
549
584
|
self.allBatches = []
|
550
585
|
rowIdMapping = {} # (rowID-columnID-columnID -> (index, evalName))
|
551
586
|
|
@@ -570,7 +605,7 @@ class LlumoClient:
|
|
570
605
|
"inactivity_timeout": 200,
|
571
606
|
"expected_results": expectedResults,
|
572
607
|
},
|
573
|
-
daemon=True
|
608
|
+
daemon=True,
|
574
609
|
)
|
575
610
|
listener_thread.start()
|
576
611
|
|
@@ -579,7 +614,9 @@ class LlumoClient:
|
|
579
614
|
|
580
615
|
# Validate API and dependencies
|
581
616
|
self.validateApiKey(evalName=evalName)
|
582
|
-
metricDependencies = checkDependency(
|
617
|
+
metricDependencies = checkDependency(
|
618
|
+
evalName, list(dataframe.columns), tocheck=_tocheck
|
619
|
+
)
|
583
620
|
if not metricDependencies["status"]:
|
584
621
|
raise LlumoAIError.dependencyError(metricDependencies["message"])
|
585
622
|
|
@@ -605,7 +642,11 @@ class LlumoClient:
|
|
605
642
|
for index, row in dataframe.iterrows():
|
606
643
|
tools = [row["tools"]] if "tools" in dataframe.columns else []
|
607
644
|
groundTruth = row.get("groundTruth", "")
|
608
|
-
messageHistory =
|
645
|
+
messageHistory = (
|
646
|
+
[row["messageHistory"]]
|
647
|
+
if "messageHistory" in dataframe.columns
|
648
|
+
else []
|
649
|
+
)
|
609
650
|
promptTemplate = prompt_template
|
610
651
|
keys = re.findall(r"{{(.*?)}}", promptTemplate)
|
611
652
|
|
@@ -615,7 +656,9 @@ class LlumoClient:
|
|
615
656
|
inputDict = {key: row[key] for key in keys if key in row}
|
616
657
|
output = row.get(outputColName, "")
|
617
658
|
|
618
|
-
activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
|
659
|
+
activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
|
660
|
+
"-", ""
|
661
|
+
)
|
619
662
|
rowID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
|
620
663
|
columnID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
|
621
664
|
|
@@ -661,7 +704,9 @@ class LlumoClient:
|
|
661
704
|
else:
|
662
705
|
if promptTemplate:
|
663
706
|
tempObj = {key: value}
|
664
|
-
promptTemplate = getInputPopulatedPrompt(
|
707
|
+
promptTemplate = getInputPopulatedPrompt(
|
708
|
+
promptTemplate, tempObj
|
709
|
+
)
|
665
710
|
else:
|
666
711
|
query += f" {key}: {value}, "
|
667
712
|
|
@@ -669,10 +714,16 @@ class LlumoClient:
|
|
669
714
|
for key, value in inputDict.items():
|
670
715
|
context += f" {key}: {value}, "
|
671
716
|
|
672
|
-
templateData["processData"]["executionDependency"][
|
673
|
-
|
717
|
+
templateData["processData"]["executionDependency"][
|
718
|
+
"context"
|
719
|
+
] = context.strip()
|
720
|
+
templateData["processData"]["executionDependency"][
|
721
|
+
"query"
|
722
|
+
] = query.strip()
|
674
723
|
if promptTemplate and not query.strip():
|
675
|
-
templateData["processData"]["executionDependency"][
|
724
|
+
templateData["processData"]["executionDependency"][
|
725
|
+
"query"
|
726
|
+
] = promptTemplate
|
676
727
|
|
677
728
|
currentBatch.append(templateData)
|
678
729
|
if len(currentBatch) == 10:
|
@@ -682,31 +733,33 @@ class LlumoClient:
|
|
682
733
|
if currentBatch:
|
683
734
|
self.allBatches.append(currentBatch)
|
684
735
|
|
685
|
-
|
686
|
-
|
736
|
+
for batch in tqdm(
|
737
|
+
self.allBatches,
|
738
|
+
desc="Processing Batches",
|
739
|
+
unit="batch",
|
740
|
+
colour="magenta",
|
741
|
+
ascii=False,
|
742
|
+
):
|
687
743
|
try:
|
688
744
|
self.postBatch(batch=batch, workspaceID=workspaceID)
|
689
|
-
|
745
|
+
time.sleep(3)
|
690
746
|
except Exception as e:
|
691
747
|
print(f"Error posting batch: {e}")
|
692
748
|
raise
|
693
749
|
|
694
|
-
|
695
750
|
# Wait for results
|
696
|
-
|
751
|
+
time.sleep(3)
|
697
752
|
listener_thread.join()
|
698
753
|
|
699
754
|
raw_results = self.socket.getReceivedData()
|
700
755
|
|
701
|
-
|
702
|
-
|
703
756
|
# Fix here: keep full keys, do not split keys
|
704
|
-
|
705
|
-
|
706
|
-
|
757
|
+
received_rowIDs = {key for item in raw_results for key in item.keys()}
|
758
|
+
expected_rowIDs = set(rowIdMapping.keys())
|
759
|
+
missing_rowIDs = expected_rowIDs - received_rowIDs
|
707
760
|
# print("All expected keys:", expected_rowIDs)
|
708
761
|
# print("All received keys:", received_rowIDs)
|
709
|
-
|
762
|
+
print("Missing keys:", len(missing_rowIDs))
|
710
763
|
|
711
764
|
# Initialize dataframe columns for each eval
|
712
765
|
for eval in evals:
|
@@ -727,26 +780,30 @@ class LlumoClient:
|
|
727
780
|
if createExperiment:
|
728
781
|
pd.set_option("future.no_silent_downcasting", True)
|
729
782
|
df = dataframe.fillna("Some error occured").astype(object)
|
730
|
-
if createPlayground(
|
731
|
-
|
783
|
+
if createPlayground(
|
784
|
+
email,
|
785
|
+
workspaceID,
|
786
|
+
df,
|
787
|
+
promptText=prompt_template,
|
788
|
+
definationMapping=self.definationMapping,
|
789
|
+
outputColName=outputColName,
|
790
|
+
):
|
732
791
|
print(
|
733
|
-
"Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
|
792
|
+
"Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
|
793
|
+
)
|
734
794
|
else:
|
735
795
|
return dataframe
|
736
796
|
|
737
797
|
def run_sweep(
|
738
|
-
|
739
|
-
|
740
|
-
|
741
|
-
|
742
|
-
|
743
|
-
|
744
|
-
|
745
|
-
|
746
|
-
) -> pd.DataFrame:
|
747
|
-
|
748
|
-
|
749
|
-
|
798
|
+
self,
|
799
|
+
templates: List[str],
|
800
|
+
dataset: Dict[str, List[str]],
|
801
|
+
model_aliases: List[AVAILABLEMODELS],
|
802
|
+
apiKey: str,
|
803
|
+
evals=["Response Correctness"],
|
804
|
+
toEvaluate: bool = False,
|
805
|
+
createExperiment: bool = False,
|
806
|
+
) -> pd.DataFrame:
|
750
807
|
|
751
808
|
self.validateApiKey(evalName="")
|
752
809
|
workspaceID = self.workspaceID
|
@@ -772,7 +829,9 @@ class LlumoClient:
|
|
772
829
|
for i, model in enumerate(model_aliases, 1):
|
773
830
|
try:
|
774
831
|
provider = getProviderFromModel(model)
|
775
|
-
response = executor.execute(
|
832
|
+
response = executor.execute(
|
833
|
+
provider, model.value, prompt, apiKey
|
834
|
+
)
|
776
835
|
outputKey = f"output_{i}"
|
777
836
|
row[outputKey] = response
|
778
837
|
except Exception as e:
|
@@ -780,14 +839,11 @@ class LlumoClient:
|
|
780
839
|
|
781
840
|
results.append(row)
|
782
841
|
|
783
|
-
|
784
|
-
|
785
842
|
df = pd.DataFrame(results)
|
786
843
|
|
787
|
-
|
788
|
-
if toEvaluate==True:
|
844
|
+
if toEvaluate == True:
|
789
845
|
dfWithEvals = df.copy()
|
790
|
-
for i, model in enumerate(model_aliases,1):
|
846
|
+
for i, model in enumerate(model_aliases, 1):
|
791
847
|
outputColName = f"output_{i}"
|
792
848
|
try:
|
793
849
|
res = self.evaluateMultiple(
|
@@ -797,7 +853,7 @@ class LlumoClient:
|
|
797
853
|
outputColName=outputColName,
|
798
854
|
_tocheck=False,
|
799
855
|
)
|
800
|
-
|
856
|
+
|
801
857
|
# Rename all new columns with _i+1 (e.g., _1, _2)
|
802
858
|
for evalMetric in evals:
|
803
859
|
scoreCol = f"{evalMetric}"
|
@@ -808,10 +864,10 @@ class LlumoClient:
|
|
808
864
|
res = res.rename(columns={reasonCol: f"{reasonCol}_{i}"})
|
809
865
|
|
810
866
|
# Drop duplicated columns from df (like prompt, variables, etc.)
|
811
|
-
newCols = [
|
867
|
+
newCols = [
|
868
|
+
col for col in res.columns if col not in dfWithEvals.columns
|
869
|
+
]
|
812
870
|
dfWithEvals = pd.concat([dfWithEvals, res[newCols]], axis=1)
|
813
|
-
|
814
|
-
|
815
871
|
|
816
872
|
except Exception as e:
|
817
873
|
print(f"Evaluation failed for model {model.value}: {str(e)}")
|
@@ -819,22 +875,31 @@ class LlumoClient:
|
|
819
875
|
if createExperiment:
|
820
876
|
pd.set_option("future.no_silent_downcasting", True)
|
821
877
|
dfWithEvals = dfWithEvals.fillna("Some error occurred")
|
822
|
-
if createPlayground(
|
823
|
-
|
824
|
-
|
878
|
+
if createPlayground(
|
879
|
+
email,
|
880
|
+
workspaceID,
|
881
|
+
dfWithEvals,
|
882
|
+
promptText=templates[0],
|
883
|
+
definationMapping=self.definationMapping,
|
884
|
+
):
|
885
|
+
|
886
|
+
print(
|
887
|
+
"Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
|
888
|
+
)
|
825
889
|
else:
|
826
890
|
return dfWithEvals
|
827
891
|
else:
|
828
|
-
if createExperiment==True:
|
892
|
+
if createExperiment == True:
|
829
893
|
pd.set_option("future.no_silent_downcasting", True)
|
830
894
|
df = df.fillna("Some error occurred")
|
831
895
|
|
832
896
|
if createPlayground(email, workspaceID, df, promptText=templates[0]):
|
833
|
-
print(
|
834
|
-
|
897
|
+
print(
|
898
|
+
"Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
|
899
|
+
)
|
900
|
+
else:
|
835
901
|
return df
|
836
902
|
|
837
|
-
|
838
903
|
# this function generates an output using llm and tools and evaluate that output
|
839
904
|
def evaluateAgents(
|
840
905
|
self,
|
@@ -856,22 +921,21 @@ class LlumoClient:
|
|
856
921
|
toolResponseDf = LlumoAgentExecutor.run(
|
857
922
|
dataframe, agents, model=model, model_api_key=model_api_key
|
858
923
|
)
|
859
|
-
|
860
924
|
|
861
925
|
# for eval in evals:
|
862
|
-
|
863
|
-
|
864
|
-
|
865
|
-
|
866
|
-
|
867
|
-
|
868
|
-
|
926
|
+
# Perform evaluation
|
927
|
+
# toolResponseDf = self.evaluate(
|
928
|
+
# toolResponseDf.to_dict(orient = "records"),
|
929
|
+
# eval=eval,
|
930
|
+
# prompt_template=prompt_template,
|
931
|
+
# createExperiment=False,
|
932
|
+
# )
|
869
933
|
toolResponseDf = self.evaluateMultiple(
|
870
|
-
toolResponseDf.to_dict(orient
|
934
|
+
toolResponseDf.to_dict(orient="records"),
|
871
935
|
evals=evals,
|
872
936
|
prompt_template=prompt_template,
|
873
937
|
createExperiment=False,
|
874
|
-
|
938
|
+
)
|
875
939
|
|
876
940
|
if createExperiment:
|
877
941
|
pd.set_option("future.no_silent_downcasting", True)
|
@@ -906,16 +970,16 @@ class LlumoClient:
|
|
906
970
|
# toolResponseDf.to_dict(orient = "records"), eval=eval, prompt_template="Give answer for the given query: {{query}}",outputColName=outputColName
|
907
971
|
# )
|
908
972
|
toolResponseDf = self.evaluateMultiple(
|
909
|
-
toolResponseDf.to_dict(orient
|
910
|
-
eval=evals,
|
911
|
-
|
973
|
+
toolResponseDf.to_dict(orient="records"),
|
974
|
+
eval=evals,
|
975
|
+
prompt_template="Give answer for the given query: {{query}}",
|
976
|
+
outputColName=outputColName,
|
912
977
|
)
|
913
978
|
return toolResponseDf
|
914
979
|
|
915
980
|
except Exception as e:
|
916
981
|
raise e
|
917
982
|
|
918
|
-
|
919
983
|
def runDataStream(
|
920
984
|
self,
|
921
985
|
data,
|
@@ -939,11 +1003,11 @@ class LlumoClient:
|
|
939
1003
|
)
|
940
1004
|
# print(f"Connected with socket ID: {socketID}")
|
941
1005
|
rowIdMapping = {}
|
942
|
-
|
943
|
-
|
1006
|
+
|
1007
|
+
# print(f"Validating API key...")
|
944
1008
|
self.validateApiKey()
|
945
|
-
|
946
|
-
|
1009
|
+
# print(f"API key validation successful. Hits available: {self.hitsAvailable}")
|
1010
|
+
|
947
1011
|
# check for available hits and trial limit
|
948
1012
|
userHits = checkUserHits(
|
949
1013
|
self.workspaceID,
|
@@ -1071,7 +1135,13 @@ class LlumoClient:
|
|
1071
1135
|
pd.set_option("future.no_silent_downcasting", True)
|
1072
1136
|
df = dataframe.fillna("Some error occured").astype(object)
|
1073
1137
|
|
1074
|
-
if createPlayground(
|
1138
|
+
if createPlayground(
|
1139
|
+
email,
|
1140
|
+
workspaceID,
|
1141
|
+
df,
|
1142
|
+
queryColName=queryColName,
|
1143
|
+
dataStreamName=streamId,
|
1144
|
+
):
|
1075
1145
|
print(
|
1076
1146
|
"Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
|
1077
1147
|
)
|
@@ -1091,6 +1161,52 @@ class LlumoClient:
|
|
1091
1161
|
except Exception as e:
|
1092
1162
|
raise "Some error ocuured please check your API key"
|
1093
1163
|
|
1164
|
+
def upload(self, file_path):
|
1165
|
+
|
1166
|
+
workspaceID = None
|
1167
|
+
email = None
|
1168
|
+
socketID = self.socket.connect(timeout=150)
|
1169
|
+
self.allBatches = []
|
1170
|
+
|
1171
|
+
# Wait for socket connection
|
1172
|
+
max_wait_secs = 20
|
1173
|
+
waited_secs = 0
|
1174
|
+
while not self.socket._connection_established.is_set():
|
1175
|
+
time.sleep(0.1)
|
1176
|
+
waited_secs += 0.1
|
1177
|
+
if waited_secs >= max_wait_secs:
|
1178
|
+
raise RuntimeError("Timeout waiting for server connection")
|
1179
|
+
|
1180
|
+
try:
|
1181
|
+
self.validateApiKey()
|
1182
|
+
except Exception as e:
|
1183
|
+
if hasattr(e, "response") and getattr(e, "response", None) is not None:
|
1184
|
+
pass
|
1185
|
+
raise
|
1186
|
+
|
1187
|
+
# Get file extension
|
1188
|
+
_, ext = os.path.splitext(file_path)
|
1189
|
+
ext = ext.lower()
|
1190
|
+
|
1191
|
+
# Supported formats
|
1192
|
+
try:
|
1193
|
+
if ext == ".csv":
|
1194
|
+
df = pd.read_csv(file_path)
|
1195
|
+
elif ext in [".xlsx", ".xls"]:
|
1196
|
+
df = pd.read_excel(file_path)
|
1197
|
+
elif ext == ".json":
|
1198
|
+
df = pd.read_json(file_path)
|
1199
|
+
elif ext == ".parquet":
|
1200
|
+
df = pd.read_parquet(file_path)
|
1201
|
+
else:
|
1202
|
+
raise ValueError(f"Unsupported file format: {ext}")
|
1203
|
+
|
1204
|
+
# If successfully loaded, call createPlayground
|
1205
|
+
createPlayground(self.email, self.workspaceID, df, _uploadViaSDK= True)
|
1206
|
+
|
1207
|
+
except Exception as e:
|
1208
|
+
print(f"Error: {e}")
|
1209
|
+
|
1094
1210
|
|
1095
1211
|
class SafeDict(dict):
|
1096
1212
|
def __missing__(self, key):
|
llumo/helpingFuntions.py
CHANGED
@@ -213,7 +213,7 @@ def deleteColumnListInPlayground(workspaceID: str, playgroundID: str):
|
|
213
213
|
return None
|
214
214
|
|
215
215
|
|
216
|
-
def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColName=None,outputColName= "output",dataStreamName=None,definationMapping=None):
|
216
|
+
def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColName=None,outputColName= "output",dataStreamName=None,definationMapping=None,uploadViaSDK = False):
|
217
217
|
if len(dataframe) > 100:
|
218
218
|
dataframe = dataframe.head(100)
|
219
219
|
print("⚠️ Dataframe truncated to 100 rows for upload.")
|
@@ -238,7 +238,7 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
|
|
238
238
|
columnIDMapping[col] = columnID
|
239
239
|
|
240
240
|
|
241
|
-
if col.startswith('output'):
|
241
|
+
if col.startswith('output') and uploadViaSDK == False:
|
242
242
|
# For output columns, create the prompt template with promptText
|
243
243
|
if promptText:
|
244
244
|
# Extract variables from promptText and set them as dependencies
|
@@ -277,7 +277,7 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
|
|
277
277
|
"order": indx,
|
278
278
|
}
|
279
279
|
|
280
|
-
elif col.startswith('Data '):
|
280
|
+
elif col.startswith('Data ') :
|
281
281
|
if queryColName and dataStreamName:
|
282
282
|
dependencies = []
|
283
283
|
dependencies.append(columnIDMapping[queryColName])
|
@@ -291,7 +291,7 @@ def createColumn(workspaceID, dataframe, playgroundID, promptText=None,queryColN
|
|
291
291
|
"type": "DATA_STREAM",
|
292
292
|
"order": indx}
|
293
293
|
|
294
|
-
elif col in allEvals:
|
294
|
+
elif col in allEvals and uploadViaSDK == False:
|
295
295
|
|
296
296
|
dependencies = []
|
297
297
|
variables = re.findall(r'{{(.*?)}}', promptText)
|
@@ -440,11 +440,11 @@ def uploadRowsInDBPlayground(payload):
|
|
440
440
|
return None
|
441
441
|
|
442
442
|
|
443
|
-
def createPlayground(email, workspaceID, df, promptText=None,queryColName=None,dataStreamName=None,definationMapping=None,outputColName="output"):
|
443
|
+
def createPlayground(email, workspaceID, df, promptText=None,queryColName=None,dataStreamName=None,definationMapping=None,outputColName="output",_uploadViaSDK = False):
|
444
444
|
|
445
445
|
playgroundId = str(createEvalPlayground(email=email, workspaceID=workspaceID))
|
446
446
|
payload1, payload2 = createColumn(
|
447
|
-
workspaceID=workspaceID, dataframe=df, playgroundID=playgroundId, promptText=promptText,queryColName=queryColName,dataStreamName=dataStreamName,definationMapping=definationMapping,outputColName=outputColName
|
447
|
+
workspaceID=workspaceID, dataframe=df, playgroundID=playgroundId, promptText=promptText,queryColName=queryColName,dataStreamName=dataStreamName,definationMapping=definationMapping,outputColName=outputColName,uploadViaSDK=_uploadViaSDK
|
448
448
|
)
|
449
449
|
|
450
450
|
|
llumo/sockets.py
CHANGED
@@ -17,8 +17,8 @@ class LlumoSocketClient:
|
|
17
17
|
|
18
18
|
# Initialize client
|
19
19
|
self.sio = socketio.Client(
|
20
|
-
logger=
|
21
|
-
engineio_logger=
|
20
|
+
logger=True,
|
21
|
+
engineio_logger=True,
|
22
22
|
reconnection=True,
|
23
23
|
reconnection_attempts=10,
|
24
24
|
reconnection_delay=1,
|
@@ -26,6 +26,7 @@ class LlumoSocketClient:
|
|
26
26
|
|
27
27
|
@self.sio.on("connect")
|
28
28
|
def on_connect():
|
29
|
+
self.sio.emit("ready")
|
29
30
|
# print("Socket connection established")
|
30
31
|
self._connected = True
|
31
32
|
# Don't set connection_established yet - wait for server confirmation
|
@@ -37,21 +38,27 @@ class LlumoSocketClient:
|
|
37
38
|
# f"Server acknowledged connection with 'connection-established' event: {data}"
|
38
39
|
# )
|
39
40
|
if isinstance(data, dict) and "socketId" in data:
|
41
|
+
self.sio.emit("ready")
|
40
42
|
self.server_socket_id = data["socketId"]
|
41
43
|
# print(f"Received server socket ID: {self.server_socket_id}")
|
42
44
|
self._connection_established.set()
|
43
45
|
|
44
46
|
@self.sio.on("result-update")
|
45
|
-
def on_result_update(data):
|
47
|
+
def on_result_update(data, callback=None):
|
46
48
|
with self._lock:
|
47
49
|
# print(f"Received result-update event: {data}")
|
48
50
|
self._received_data.append(data)
|
49
51
|
self._last_update_time = time.time()
|
50
52
|
|
51
53
|
# ✅ Stop if all expected results are received
|
52
|
-
if
|
54
|
+
if (
|
55
|
+
self._expected_results
|
56
|
+
and len(self._received_data) >= self._expected_results
|
57
|
+
):
|
53
58
|
# print("✅ All expected results received.")
|
54
59
|
self._listening_done.set()
|
60
|
+
if callback:
|
61
|
+
callback(True)
|
55
62
|
|
56
63
|
@self.sio.on("disconnect")
|
57
64
|
def on_disconnect():
|
@@ -81,13 +88,17 @@ class LlumoSocketClient:
|
|
81
88
|
start = time.time()
|
82
89
|
while not self.sio.connected:
|
83
90
|
if time.time() - start > timeout:
|
84
|
-
raise RuntimeError(
|
91
|
+
raise RuntimeError(
|
92
|
+
"Timed out waiting for low-level socket connection."
|
93
|
+
)
|
85
94
|
time.sleep(0.1)
|
86
95
|
# print("[DEBUG] SocketIO low-level connection established.")
|
87
96
|
|
88
97
|
# Wait for server "connection-established" event
|
89
98
|
if not self._connection_established.wait(timeout):
|
90
|
-
raise RuntimeError(
|
99
|
+
raise RuntimeError(
|
100
|
+
"Timed out waiting for connection-established event."
|
101
|
+
)
|
91
102
|
|
92
103
|
self._connected = True
|
93
104
|
self._last_update_time = time.time()
|
@@ -100,7 +111,9 @@ class LlumoSocketClient:
|
|
100
111
|
self._connected = False
|
101
112
|
raise RuntimeError(f"WebSocket connection failed: {e}")
|
102
113
|
|
103
|
-
def listenForResults(
|
114
|
+
def listenForResults(
|
115
|
+
self, min_wait=30, max_wait=300, inactivity_timeout=50, expected_results=None
|
116
|
+
):
|
104
117
|
# if not self._connected:
|
105
118
|
# raise RuntimeError("WebSocket is not connected. Call connect() first.")
|
106
119
|
|
@@ -129,8 +142,6 @@ class LlumoSocketClient:
|
|
129
142
|
self._listening_done.set()
|
130
143
|
break
|
131
144
|
|
132
|
-
|
133
|
-
|
134
145
|
timeout_thread = threading.Thread(target=timeout_watcher, daemon=True)
|
135
146
|
timeout_thread.start()
|
136
147
|
self._listening_done.wait()
|
@@ -0,0 +1,13 @@
|
|
1
|
+
llumo/__init__.py,sha256=O04b4yW1BnOvcHzxWFddAKhtdBEhBNhLdb6xgnpHH_Q,205
|
2
|
+
llumo/client.py,sha256=YmvbfyWR9YCDOFrKM0nwDMWYLGpu4RSZwbkiUJ3e78M,46162
|
3
|
+
llumo/exceptions.py,sha256=i3Qv4_g7XjRuho7-b7ybjw2bwSh_NhvICR6ZAgiLQX8,1944
|
4
|
+
llumo/execution.py,sha256=x88wQV8eL99wNN5YtjFaAMCIfN1PdfQVlAZQb4vzgQ0,1413
|
5
|
+
llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
|
6
|
+
llumo/helpingFuntions.py,sha256=f2Y-x-DbGk3E29qaJWDOsTkuqqDFl9-VQTRM490amE4,20443
|
7
|
+
llumo/models.py,sha256=YH-qAMnShmUpmKE2LQAzQdpRsaXkFSlOqMxHwU4zBUI,1560
|
8
|
+
llumo/sockets.py,sha256=-zJYRCDRwElIPr5iOFqzQxjecuLJ7mztiyYJz14pGLY,5949
|
9
|
+
llumo-0.2.14b6.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
|
10
|
+
llumo-0.2.14b6.dist-info/METADATA,sha256=2Yl4gnAXsfpJWLB6mhlza0HUE76uJY3sC1TWK7GlUu4,1521
|
11
|
+
llumo-0.2.14b6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
12
|
+
llumo-0.2.14b6.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
|
13
|
+
llumo-0.2.14b6.dist-info/RECORD,,
|
llumo-0.2.14b4.dist-info/RECORD
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
llumo/__init__.py,sha256=O04b4yW1BnOvcHzxWFddAKhtdBEhBNhLdb6xgnpHH_Q,205
|
2
|
-
llumo/client.py,sha256=Zque6TDcEBFO1tQjNMsmUpYVbGdooLiwDKDYPtJ2szY,43465
|
3
|
-
llumo/exceptions.py,sha256=i3Qv4_g7XjRuho7-b7ybjw2bwSh_NhvICR6ZAgiLQX8,1944
|
4
|
-
llumo/execution.py,sha256=x88wQV8eL99wNN5YtjFaAMCIfN1PdfQVlAZQb4vzgQ0,1413
|
5
|
-
llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
|
6
|
-
llumo/helpingFuntions.py,sha256=9aFvBNPB-AyeMs6c8YpdPckNeF95thmTqAAmYC1q4oo,20320
|
7
|
-
llumo/models.py,sha256=YH-qAMnShmUpmKE2LQAzQdpRsaXkFSlOqMxHwU4zBUI,1560
|
8
|
-
llumo/sockets.py,sha256=Rlww5z9w_Ij99Z_VzgpF93K9gSZ5YaS4A8oAY0jS1wA,5650
|
9
|
-
llumo-0.2.14b4.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
|
10
|
-
llumo-0.2.14b4.dist-info/METADATA,sha256=mmsv1MSuhdFIkxYvhhpYIhFLZKs5U2lA87ND2D8WK8A,1521
|
11
|
-
llumo-0.2.14b4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
12
|
-
llumo-0.2.14b4.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
|
13
|
-
llumo-0.2.14b4.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|