AutoRAG 0.2.11__tar.gz → 0.2.12__tar.gz

{autorag-0.2.11 → autorag-0.2.12}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.11
+Version: 0.2.12
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -242,7 +242,7 @@ Requires-Dist: uvicorn
 Requires-Dist: torch
 Requires-Dist: sentencepiece
 Requires-Dist: guidance
-Requires-Dist: cohere>=5.0.0a9
+Requires-Dist: cohere<=5.6.2
 Requires-Dist: tokenlog>=0.0.2
 Requires-Dist: aiohttp
 Requires-Dist: bert_score

{autorag-0.2.11 → autorag-0.2.12}/AutoRAG.egg-info/requires.txt

@@ -20,7 +20,7 @@ uvicorn
 torch
 sentencepiece
 guidance
-cohere>=5.0.0a9
+cohere<=5.6.2
 tokenlog>=0.0.2
 aiohttp
 bert_score

{autorag-0.2.11 → autorag-0.2.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.11
+Version: 0.2.12
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -242,7 +242,7 @@ Requires-Dist: uvicorn
 Requires-Dist: torch
 Requires-Dist: sentencepiece
 Requires-Dist: guidance
-Requires-Dist: cohere>=5.0.0a9
+Requires-Dist: cohere<=5.6.2
 Requires-Dist: tokenlog>=0.0.2
 Requires-Dist: aiohttp
 Requires-Dist: bert_score

autorag-0.2.12/autorag/VERSION

@@ -0,0 +1 @@
+0.2.12

{autorag-0.2.11 → autorag-0.2.12}/autorag/evaluation/metric/generation.py

@@ -6,13 +6,13 @@ from typing import List, Optional
 
 import evaluate
 import pandas as pd
-from sacrebleu.metrics.bleu import BLEU
 import torch
 from llama_index.core.embeddings import BaseEmbedding
 from llama_index.embeddings.openai import OpenAIEmbedding
 from openai import AsyncOpenAI
 from rouge_score import tokenizers
 from rouge_score.rouge_scorer import RougeScorer
+from sacrebleu.metrics.bleu import BLEU
 
 from autorag import embedding_models
 from autorag.evaluation.metric.util import calculate_cosine_similarity
@@ -65,7 +65,9 @@ def huggingface_evaluate(instance, key: str,
 
 
 @convert_inputs_to_list
-def bleu(generation_gt: List[List[str]], generations: [str], tokenize: str|None = None, smooth_method: str = 'exp', smooth_value: Optional[float] = None, max_ngram_order: int = 4, trg_lang: str = '', **kwargs) -> List[float]:
+def bleu(generation_gt: List[List[str]], generations: [str], tokenize: Optional[str] = None,
+         smooth_method: str = 'exp', smooth_value: Optional[float] = None, max_ngram_order: int = 4,
+         trg_lang: str = '', **kwargs) -> List[float]:
     """
     Computes the BLEU metric given pred and ground-truth.
 
@@ -79,9 +81,10 @@ def bleu(generation_gt: List[List[str]], generations: [str], tokenize: str|None
         Because it can be a multiple ground truth.
     :param generations: A list of generations that LLM generated.
     """
-    bleu = BLEU(tokenize=tokenize, smooth_method=smooth_method, smooth_value=smooth_value, max_ngram_order=max_ngram_order, trg_lang=trg_lang, **kwargs)
+    bleu_instance = BLEU(tokenize=tokenize, smooth_method=smooth_method, smooth_value=smooth_value,
+                         max_ngram_order=max_ngram_order, trg_lang=trg_lang, **kwargs)
 
-    result = list(map(lambda x: bleu.sentence_score(x[0], x[1]).score, zip(generations, generation_gt)))
+    result = list(map(lambda x: bleu_instance.sentence_score(x[0], x[1]).score, zip(generations, generation_gt)))
     return result
 
 
@@ -319,4 +322,8 @@ def bert_score(generation_gt: List[List[str]], generations: List[str],
     df['bert_score'] = evaluator.compute(predictions=df['prediction'].tolist(),
                                          references=df['reference'].tolist(),
                                          lang=lang, nthreads=n_threads, batch_size=batch)['f1']
+
+    del evaluator
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
     return df.groupby(level=0)['bert_score'].max().tolist()

{autorag-0.2.11 → autorag-0.2.12}/requirements.txt

@@ -20,7 +20,7 @@ uvicorn  # for api server
 torch  # for monot5 reranker
 sentencepiece  # for monot5 reranker
 guidance # for qa data creation
-cohere>=5.0.0a9 # for cohere services
+cohere<=5.6.2 # for cohere services
 tokenlog>=0.0.2 # for token logging
 aiohttp # for async http requests
 bert_score # for bert score

{autorag-0.2.11 → autorag-0.2.12}/tests/autorag/evaluate/metric/test_generation_metric.py

@@ -47,6 +47,7 @@ def ko_base_test_generation_metrics(func, solution, **kwargs):
     assert all(list(map(lambda x: x[0] == pytest.approx(x[1], 0.001),
                         zip(scores, solution))))
 
+
 def test_bleu():
     base_test_generation_metrics(bleu, [51.1507, 23.5783, 100.0], lowercase=True)
 

autorag-0.2.11/autorag/VERSION

@@ -1 +0,0 @@
-0.2.11