llumo 0.2.11__py3-none-any.whl → 0.2.12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
llumo/client.py CHANGED
@@ -153,6 +153,7 @@ class LlumoClient:
153
153
 
154
154
  return uniqueResults
155
155
 
156
+ # this function allows the users to run exactl one eval at a time
156
157
  def evaluate(
157
158
  self,
158
159
  dataframe,
@@ -369,31 +370,46 @@ class LlumoClient:
369
370
  else:
370
371
  return dataframe
371
372
 
373
+ # this function allows the users to run multiple evals at once
374
+ def evaluateMultiple(
375
+ self,
376
+ dataframe,
377
+ eval=["Response Completeness"],
378
+ prompt_template="",
379
+ outputColName="output",
380
+ createExperiment: bool = False,
381
+ ):
382
+ resultdf = dataframe.copy()
383
+ for evalName in eval:
384
+ resultdf = self.evaluate(dataframe = resultdf,eval=evalName,prompt_template=prompt_template,outputColName=outputColName,createExperiment = False)
385
+
386
+ if createExperiment:
387
+ pd.set_option("future.no_silent_downcasting", True)
388
+ df = resultdf.fillna("Some error occured").astype(object)
389
+
390
+ if createPlayground(self.email, self.workspaceID, df):
391
+ print(
392
+ "Your data has been saved in the Llumo Experiment. Visit https://app.llumo.ai/evallm to see the results."
393
+ )
394
+ else:
395
+ return resultdf
396
+
397
+
372
398
  def evaluateCompressor(self, dataframe, prompt_template):
373
399
  results = []
374
400
 
375
401
  try:
376
- # Connect to socket first
377
- # print("Connecting to socket server...")
378
402
  socketID = self.socket.connect(timeout=150)
379
-
380
- # Ensure full connection before proceeding
381
403
  max_wait_secs = 20
382
404
  waited_secs = 0
383
405
  while not self.socket._connection_established.is_set():
384
406
  time.sleep(0.1)
385
407
  waited_secs += 0.1
386
408
  if waited_secs >= max_wait_secs:
387
- raise RuntimeError(
388
- "Timeout waiting for server 'connection-established' event."
389
- )
390
-
391
- # print(f"Connected with socket ID: {socketID}")
409
+ raise RuntimeError("Timeout waiting for server 'connection-established' event.")
392
410
 
393
411
  try:
394
- # print(f"Validating API key...")
395
412
  self.validateApiKey()
396
- # print(f"API key validation successful. Hits available: {self.hitsAvailable}")
397
413
  except Exception as e:
398
414
  print(f"Error during API key validation: {str(e)}")
399
415
  if hasattr(e, "response") and getattr(e, "response", None) is not None:
@@ -401,61 +417,42 @@ class LlumoClient:
401
417
  print(f"Response content: {e.response.text[:500]}...")
402
418
  raise
403
419
 
404
- # check for available hits and trial limit
405
- userHits = checkUserHits(
406
- self.workspaceID,
407
- self.hasSubscribed,
408
- self.trialEndDate,
409
- self.subscriptionEndDate,
410
- self.hitsAvailable,
411
- len(dataframe),
412
- )
420
+ userHits = checkUserHits(self.workspaceID, self.hasSubscribed, self.trialEndDate, self.subscriptionEndDate,
421
+ self.hitsAvailable, len(dataframe))
413
422
 
414
- # do not proceed if subscription or trial limit has exhausted
415
423
  if not userHits["success"]:
416
424
  raise LlumoAIError.InsufficientCredits(userHits["message"])
417
425
 
418
- # if self.hitsAvailable == 0 or len(dataframe) > self.hitsAvailable:
419
- # raise LlumoAIError.InsufficientCredits()
420
-
421
426
  model = "GPT_4"
422
427
  provider = "OPENAI"
423
428
  evalType = "LLUMO"
424
429
  workspaceID = self.workspaceID
425
430
  email = self.email
426
- # Prepare all batches before sending
427
- # print("Preparing batches...")
428
431
  self.allBatches = []
429
432
  currentBatch = []
430
-
433
+ rowIdMapping = {}
431
434
  for index, row in dataframe.iterrows():
432
435
  promptTemplate = prompt_template
433
-
434
- # extracting the placeholders from the prompt template
435
436
  keys = re.findall(r"{{(.*?)}}", promptTemplate)
436
437
  inputDict = {key: row[key] for key in keys if key in row}
437
438
 
438
439
  if not all([ky in dataframe.columns for ky in keys]):
439
440
  raise LlumoAIError.InvalidPromptTemplate()
440
441
 
441
- activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
442
- "-", ""
443
- )
442
+ activePlayground = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
444
443
  rowID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
445
444
  columnID = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
446
445
 
447
- compressed_prompt_id = (
448
- f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
449
- )
450
- compressed_prompt_output_id = (
451
- f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
452
- )
446
+ compressed_prompt_id = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
447
+ compressed_prompt_output_id = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
453
448
  cost_id = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
454
- cost_saving_id = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace(
455
- "-", ""
456
- )
449
+ cost_saving_id = f"{int(time.time() * 1000)}{uuid.uuid4()}".replace("-", "")
450
+
451
+ rowDataDict = {}
452
+ for col in dataframe.columns:
453
+ val = row[col]
454
+ rowDataDict[col] = {"type": "VARIABLE", "value": str(val)}
457
455
 
458
- # Use the server-provided socket ID here
459
456
  templateData = {
460
457
  "processID": getProcessID(),
461
458
  "socketID": socketID,
@@ -469,13 +466,10 @@ class LlumoClient:
469
466
  "compressed_prompt": compressed_prompt_id,
470
467
  "compressed_prompt_output": compressed_prompt_output_id,
471
468
  "cost": cost_id,
472
- "cost_saving": cost_saving_id,
469
+ "cost_saving": cost_saving_id
473
470
  },
474
471
  "processData": {
475
- "rowData": {
476
- "query": {"type": "VARIABLE", "value": ""},
477
- "context": {"type": "VARIABLE", "value": ""},
478
- },
472
+ "rowData": rowDataDict,
479
473
  "dependency": list(inputDict.keys()),
480
474
  "dependencyMapping": {ky: ky for ky in list(inputDict.keys())},
481
475
  "provider": provider,
@@ -485,111 +479,71 @@ class LlumoClient:
485
479
  "compressed_prompt": compressed_prompt_id,
486
480
  "compressed_prompt_output": compressed_prompt_output_id,
487
481
  "cost": cost_id,
488
- "cost_saving": cost_saving_id,
489
- },
482
+ "cost_saving": cost_saving_id
483
+ }
490
484
  },
491
485
  "workspaceID": workspaceID,
492
486
  "email": email,
493
- "playgroundID": activePlayground,
487
+ "playgroundID": activePlayground
494
488
  }
495
489
 
496
- # Build query/context from input
497
- query = ""
498
- context = ""
499
-
500
- for key, value in inputDict.items():
501
- if isinstance(value, str):
502
- length = len(value.split()) * 1.5
503
- if length > 50:
504
- context += f" {key}: {value}, "
505
- else:
506
- if promptTemplate:
507
- populatedPrompt = getInputPopulatedPrompt(
508
- promptTemplate, {key: value}
509
- )
510
- query += f"{populatedPrompt} "
511
- else:
512
- query += f" {key}: {value}, "
513
-
514
- if not context.strip():
515
- for key, value in inputDict.items():
516
- context += f" {key}: {value}, "
517
-
518
- templateData["processData"]["rowData"]["context"][
519
- "value"
520
- ] = context.strip()
521
- templateData["processData"]["rowData"]["query"]["value"] = query.strip()
522
-
523
- if promptTemplate and not query.strip():
524
- templateData["processData"]["rowData"]["query"][
525
- "value"
526
- ] = promptTemplate
527
-
490
+ rowIdMapping[rowID] = index
491
+ # print("__________________________TEMPLATE__________________________________")
528
492
  # print(templateData)
493
+
529
494
  currentBatch.append(templateData)
530
495
 
531
- if len(currentBatch) == 10 or index == len(dataframe) - 1:
532
- self.allBatches.append(currentBatch)
533
- currentBatch = []
496
+ if len(currentBatch) == 10 or index == len(dataframe) - 1:
497
+ self.allBatches.append(currentBatch)
498
+ currentBatch = []
534
499
 
535
- # Post all batches
536
500
  total_items = sum(len(batch) for batch in self.allBatches)
537
- # print(f"Posting {len(self.allBatches)} batches ({total_items} items total)")
538
501
 
539
502
  for cnt, batch in enumerate(self.allBatches):
540
- # print(f"Posting batch {cnt + 1}/{len(self.allBatches)} for eval '{eval}'")
541
503
  try:
542
504
  self.postBatch(batch=batch, workspaceID=workspaceID)
543
- # print(f"Batch {cnt + 1} posted successfully")
544
505
  except Exception as e:
545
506
  print(f"Error posting batch {cnt + 1}: {str(e)}")
546
507
  continue
547
-
548
- # Small delay between batches to prevent overwhelming the server
549
508
  time.sleep(1)
550
509
 
551
- # updating the dict for row column mapping
552
510
  self.AllProcessMapping()
553
- # Calculate a reasonable timeout based on the data size
554
511
  timeout = max(60, min(600, total_items * 10))
555
- # print(f"All batches posted. Waiting up to {timeout} seconds for results...")
556
-
557
- # Listen for results
558
- self.socket.listenForResults(
559
- min_wait=20,
560
- max_wait=timeout,
561
- inactivity_timeout=30,
562
- expected_results=None,
563
- )
512
+ self.socket.listenForResults(min_wait=20, max_wait=timeout, inactivity_timeout=30, expected_results=None)
564
513
 
565
- # Get results for this evaluation
566
- eval_results = self.socket.getReceivedData()
567
- # print(f"Received {len(eval_results)} results for evaluation '{eval}'")
568
-
569
- # Add these results to our overall results
570
- results = self.finalResp(eval_results)
571
- print(f"======= Completed evaluation: {eval} =======\n")
572
-
573
- # print("All evaluations completed successfully")
514
+ results = self.socket.getReceivedData()
515
+ # results = self.finalResp(eval_results)
516
+ # print(f"======= Completed evaluation: {eval} =======\n")
574
517
 
575
518
  except Exception as e:
576
519
  print(f"Error during evaluation: {e}")
577
520
  raise
578
521
  finally:
579
- # Always disconnect the socket when done
580
522
  try:
581
523
  self.socket.disconnect()
582
- # print("Socket disconnected")
583
524
  except Exception as e:
584
525
  print(f"Error disconnecting socket: {e}")
585
526
 
586
- compressed_prompt, compressed_prompt_output, cost, cost_saving = (
587
- costColumnMapping(results, self.processMapping)
588
- )
589
- dataframe["compressed_prompt"] = compressed_prompt
590
- dataframe["compressed_prompt_output"] = compressed_prompt_output
591
- dataframe["cost"] = cost
592
- dataframe["cost_saving"] = cost_saving
527
+ dataframe["Compressed Input"] = None
528
+ for records in results:
529
+ for compound_key, value in records.items():
530
+ # for compound_key, value in item['data'].items():
531
+ rowID = compound_key.split('-')[0]
532
+ # looking for the index of each rowID , in the original dataframe
533
+ if rowID in rowIdMapping:
534
+ index = rowIdMapping[rowID]
535
+
536
+ dataframe.at[index, "Compressed Input"] = value["value"]
537
+
538
+ else:
539
+ pass
540
+ # print(f"⚠️ Warning: Could not find rowID {rowID} in mapping")
541
+
542
+ # compressed_prompt, compressed_prompt_output, cost, cost_saving = costColumnMapping(results, self.processMapping)
543
+ # dataframe["compressed_prompt"] = compressed_prompt
544
+ # dataframe["compressed_prompt_output"] = compressed_prompt_output
545
+ # dataframe["cost"] = cost
546
+ # dataframe["cost_saving"] = cost_saving
593
547
 
594
548
  return dataframe
595
549
 
@@ -644,7 +598,7 @@ class LlumoClient:
644
598
  df = pd.DataFrame(results)
645
599
  if toEvaluate:
646
600
 
647
- res = self.evaluate(df, eval=eval, prompt_template=str(templates[0]))
601
+ res = self.evaluateMultiple(df, eval=eval, prompt_template=str(templates[0]))
648
602
 
649
603
  if createExperiment:
650
604
  pd.set_option("future.no_silent_downcasting", True)
@@ -667,6 +621,7 @@ class LlumoClient:
667
621
  else:
668
622
  return df
669
623
 
624
+ # this function generates an output using llm and tools and evaluate that output
670
625
  def evaluateAgents(
671
626
  self,
672
627
  dataframe,
@@ -708,6 +663,7 @@ class LlumoClient:
708
663
  else:
709
664
  return toolResponseDf
710
665
 
666
+ # this function evaluate that tools output given by the user
711
667
  def evaluateAgentResponses(
712
668
  self,
713
669
  dataframe,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: llumo
3
- Version: 0.2.11
3
+ Version: 0.2.12
4
4
  Summary: Python SDK for interacting with the Llumo ai API.
5
5
  Home-page: https://www.llumo.ai/
6
6
  Author: Llumo
@@ -1,13 +1,13 @@
1
1
  llumo/__init__.py,sha256=O04b4yW1BnOvcHzxWFddAKhtdBEhBNhLdb6xgnpHH_Q,205
2
- llumo/client.py,sha256=_TpTvzt5ox5MUCZ5O07AY6DYrtIhtezEDAE6nG7nUjs,35632
2
+ llumo/client.py,sha256=QMasCjzRmL38RuMUHoY39_ge2nIRdSOECjj1daicd-k,34144
3
3
  llumo/exceptions.py,sha256=iCj7HhtO_ckC2EaVBdXbAudNpuMDsYmmMEV5lwynZ-E,1854
4
4
  llumo/execution.py,sha256=x88wQV8eL99wNN5YtjFaAMCIfN1PdfQVlAZQb4vzgQ0,1413
5
5
  llumo/functionCalling.py,sha256=D5jYapu1rIvdIJNUYPYMTyhQ1H-6nkwoOLMi6eekfUE,7241
6
6
  llumo/helpingFuntions.py,sha256=lG_d3lQgJj6pI7v1YdLqdPojrLCNwybKz29zXrGaL5k,9090
7
7
  llumo/models.py,sha256=YH-qAMnShmUpmKE2LQAzQdpRsaXkFSlOqMxHwU4zBUI,1560
8
8
  llumo/sockets.py,sha256=Qxxqtx3Hg07HLhA4QfcipK1ChiOYhHZBu02iA6MfYlQ,5579
9
- llumo-0.2.11.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
10
- llumo-0.2.11.dist-info/METADATA,sha256=ymWr0L9PnYitu0FnBAZ2NsJnlOxhcsUoUVA8Yn-DXoQ,1491
11
- llumo-0.2.11.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
12
- llumo-0.2.11.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
13
- llumo-0.2.11.dist-info/RECORD,,
9
+ llumo-0.2.12.dist-info/licenses/LICENSE,sha256=tF9yAcfPV9xGT3ViWmC8hPvOo8BEk4ZICbUfcEo8Dlk,182
10
+ llumo-0.2.12.dist-info/METADATA,sha256=CYu8qMtD-XZuoQzC7YPZRlNgetEQPjMT1RiMGnHdZJs,1491
11
+ llumo-0.2.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
12
+ llumo-0.2.12.dist-info/top_level.txt,sha256=d5zUTMI99llPtLRB8rtSrqELm_bOqX-bNC5IcwlDk88,6
13
+ llumo-0.2.12.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (80.8.0)
2
+ Generator: setuptools (80.9.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5