AutoRAG 0.0.2__tar.gz → 0.0.4__tar.gz

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.0.2
+Version: 0.0.4
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -310,7 +310,7 @@ For evaluation, you need to prepare just three files.
 
 There is a template for your evaluation data for using AutoRAG.
 Check out the evaluation data rule at [here]().
-Plus, you can get example datasets for testing AutoRAG at [here]().
+Plus, you can get example datasets for testing AutoRAG at [here](./sample_dataset).
 
 ### Evaluate your data to various RAG modules
 

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/AutoRAG.egg-info/SOURCES.txt

@@ -1,7 +1,6 @@
 .gitignore
 LICENSE
 README.md
-VERSION
 dev_requirements.txt
 pyproject.toml
 requirements.txt
@@ -14,6 +13,7 @@ AutoRAG.egg-info/dependency_links.txt
 AutoRAG.egg-info/entry_points.txt
 AutoRAG.egg-info/requires.txt
 AutoRAG.egg-info/top_level.txt
+autorag/VERSION
 autorag/__init__.py
 autorag/deploy.py
 autorag/evaluator.py
@@ -74,6 +74,8 @@ docs/make.bat
 docs/requirements.txt
 docs/source/conf.py
 docs/source/index.rst
+docs/source/install.md
+docs/source/tutorial.md
 docs/source/api_spec/autorag.data.rst
 docs/source/api_spec/autorag.evaluate.metric.rst
 docs/source/api_spec/autorag.evaluate.rst
@@ -89,6 +91,14 @@ docs/source/api_spec/autorag.rst
 docs/source/api_spec/autorag.schema.rst
 docs/source/api_spec/autorag.utils.rst
 docs/source/api_spec/modules.rst
+docs/source/data_creation/tutorial.md
+docs/source/modules/index.md
+docs/source/optimization/optimization.md
+docs/source/resources/samsung_sundae.jpeg
+sample_dataset/README.md
+sample_dataset/eli5/load_eli5_dataset.py
+sample_dataset/msmarco/load_msmarco_dataset.py
+sample_dataset/triviaqa/load_triviaqa_dataset.py
 tests/conftest.py
 tests/autorag/test_deploy.py
 tests/autorag/test_evaluator.py
@@ -128,6 +138,7 @@ tests/autorag/utils/test_util.py
 tests/resources/corpus_data_sample.parquet
 tests/resources/full.yaml
 tests/resources/qa_data_sample.parquet
+tests/resources/qa_test_data_sample.parquet
 tests/resources/simple.yaml
 tests/resources/test_bm25_retrieval.pkl
 tests/resources/result_project/trial.json

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.0.2
+Version: 0.0.4
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -310,7 +310,7 @@ For evaluation, you need to prepare just three files.
 
 There is a template for your evaluation data for using AutoRAG.
 Check out the evaluation data rule at [here]().
-Plus, you can get example datasets for testing AutoRAG at [here]().
+Plus, you can get example datasets for testing AutoRAG at [here](./sample_dataset).
 
 ### Evaluate your data to various RAG modules
 

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/README.md

@@ -66,7 +66,7 @@ For evaluation, you need to prepare just three files.
 
 There is a template for your evaluation data for using AutoRAG.
 Check out the evaluation data rule at [here]().
-Plus, you can get example datasets for testing AutoRAG at [here]().
+Plus, you can get example datasets for testing AutoRAG at [here](./sample_dataset).
 
 ### Evaluate your data to various RAG modules
 

AutoRAG-0.0.4/autorag/VERSION

@@ -0,0 +1 @@
+0.0.4

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/autorag/__init__.py

@@ -10,8 +10,7 @@ from llama_index.embeddings.openai import OpenAIEmbeddingModelType
 from llama_index.llms import OpenAI, Anthropic, AzureOpenAI, HuggingFaceLLM, LangChainLLM, GradientBaseModelLLM, \
     GradientModelAdapterLLM, LiteLLM, LlamaCPP, OpenAILike, OpenLLM, PaLM, PredibaseLLM, Replicate, Xinference
 
-root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-version_path = os.path.join(root_path, 'VERSION')
+version_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'VERSION')
 
 with open(version_path, 'r') as f:
     __version__ = f.read().strip()
@@ -47,7 +46,7 @@ generator_models = {
 
 rich_format = "[%(filename)s:%(lineno)s] >> %(message)s"
 logging.basicConfig(
-    level="NOTSET",
+    level="INFO",
     format=rich_format,
     handlers=[RichHandler(rich_tracebacks=True)]
 )

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/autorag/deploy.py

@@ -27,6 +27,20 @@ def extract_node_line_names(config_dict: Dict) -> List[str]:
     return [node_line['node_line_name'] for node_line in config_dict['node_lines']]
 
 
+def extract_node_strategy(config_dict: Dict) -> Dict:
+    """
+    Extract node strategies with the given config dictionary.
+    The return value is a dictionary of node type and its strategy.
+
+    :param config_dict: The yaml configuration dict for the pipeline.
+        You can load this to access trail_folder/config.yaml.
+    :return: Key is node_type and value is strategy dict.
+    """
+    return {node['node_type']: node.get('strategy', {})
+            for node_line in config_dict['node_lines']
+            for node in node_line['nodes']}
+
+
 def summary_df_to_yaml(summary_df: pd.DataFrame, config_dict: Dict) -> Dict:
     """
     Convert trial summary dataframe to config yaml file.
@@ -41,6 +55,12 @@ def summary_df_to_yaml(summary_df: pd.DataFrame, config_dict: Dict) -> Dict:
     # summary_df columns : 'node_line_name', 'node_type', 'best_module_filename',
     #                      'best_module_name', 'best_module_params', 'best_execution_time'
     node_line_names = extract_node_line_names(config_dict)
+    node_strategies = extract_node_strategy(config_dict)
+    strategy_df = pd.DataFrame({
+        'node_type': list(node_strategies.keys()),
+        'strategy': list(node_strategies.values())
+    })
+    summary_df = summary_df.merge(strategy_df, on='node_type', how='left')
     summary_df['categorical_node_line_name'] = pd.Categorical(summary_df['node_line_name'], categories=node_line_names,
                                                               ordered=True)
     summary_df = summary_df.sort_values(by='categorical_node_line_name')
@@ -52,6 +72,7 @@ def summary_df_to_yaml(summary_df: pd.DataFrame, config_dict: Dict) -> Dict:
             'nodes': [
                 {
                     'node_type': row['node_type'],
+                    'strategy': row['strategy'],
                     'modules': [{
                         'module_type': row['best_module_name'],
                         **row['best_module_params']

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/autorag/nodes/generator/base.py

@@ -34,6 +34,7 @@ def generator_node(func):
             if llm not in generator_models:
                 raise ValueError(f"{llm} is not a valid llm name. Please check the llm name."
                                  "You can check valid llm names from autorag.generator_models.")
+            batch = kwargs.pop('batch', 16)
             llm_instance = generator_models[llm](**kwargs)
-            return func(prompts=prompts, llm=llm_instance)
+            return func(prompts=prompts, llm=llm_instance, batch=batch)
     return wrapper

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/autorag/nodes/generator/llama_index_llm.py

@@ -5,16 +5,21 @@ from llama_index.llms.base import BaseLLM
 from transformers import AutoTokenizer
 
 from autorag.nodes.generator.base import generator_node
+from autorag.utils.util import process_batch
 
 
 @generator_node
-def llama_index_llm(prompts: List[str], llm: BaseLLM) -> Tuple[List[str], List[List[int]], List[List[float]]]:
+def llama_index_llm(prompts: List[str], llm: BaseLLM, batch: int = 16) -> Tuple[List[str], List[List[int]], List[List[float]]]:
     """
     Llama Index LLM module.
     It gets the LLM instance from llama index, and returns generated text by the input prompt.
     It does not generate the right log probs, but it returns the pseudo log probs,
     which is not meant to be used for other modules.
 
+    :param prompts: A list of prompts.
+    :param llm: A llama index LLM instance.
+    :param batch: The batch size for llm.
+        Set low if you face some errors.
     :return: A tuple of three elements.
         The first element is a list of generated text.
         The second element is a list of generated text's token ids, used tokenizer is GPT2Tokenizer.
@@ -22,7 +27,7 @@ def llama_index_llm(prompts: List[str], llm: BaseLLM) -> Tuple[List[str], List[L
     """
     tasks = [llm.acomplete(prompt) for prompt in prompts]
     loop = asyncio.get_event_loop()
-    results = loop.run_until_complete(asyncio.gather(*tasks))
+    results = loop.run_until_complete(process_batch(tasks, batch_size=batch))
 
     generated_texts = list(map(lambda x: x.text, results))
     tokenizer = AutoTokenizer.from_pretrained("gpt2", use_fast=False)

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/autorag/nodes/passagecompressor/base.py

@@ -27,7 +27,7 @@ def passage_compressor_node(func):
         retrieve_scores = previous_result['retrieve_scores'].tolist()
 
         if func.__name__ == 'tree_summarize':
-            param_list = ['prompt', 'chat_prompt', 'context_window', 'num_output']
+            param_list = ['prompt', 'chat_prompt', 'context_window', 'num_output', 'batch']
             param_dict = dict(filter(lambda x: x[0] in param_list, kwargs.items()))
             kwargs_dict = dict(filter(lambda x: x[0] not in param_list, kwargs.items()))
             llm_name = kwargs_dict.pop('llm')

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/autorag/nodes/passagecompressor/tree_summarize.py

@@ -8,6 +8,7 @@ from llama_index.prompts.utils import is_chat_model
 from llama_index.response_synthesizers import TreeSummarize
 
 from autorag.nodes.passagecompressor.base import passage_compressor_node
+from autorag.utils.util import process_batch
 
 
 @passage_compressor_node
@@ -20,11 +21,12 @@ def tree_summarize(queries: List[str],
                    chat_prompt: Optional[str] = None,
                    context_window: Optional[int] = None,
                    num_output: int = 1,
+                   batch: int = 16,
                    ) -> List[str]:
     """
     Recursively merge retrieved texts and summarizes them in a bottom-up fashion.
     This function is a wrapper for llama_index.response_synthesizers.TreeSummarize.
-    For more information, visit https://docs.llamaindex.ai/en/stable/examples/response_synthesizers/tree_summarize.html.
+    For more information, visit https://docs.llamaindex.ai/en/latest/examples/response_synthesizers/tree_summarize.html.
 
     :param queries: The queries for retrieved passages.
     :param contents: The contents of retrieved passages.
@@ -45,6 +47,9 @@ def tree_summarize(queries: List[str],
         Default is None. When it is None, it will use a llama index default context window.
     :param num_output: The amount of summarization output.
         Default is 1.
+    :param batch: The batch size for llm.
+        Set low if you face some errors.
+        Default is 16.
     :return: The list of compressed texts.
     """
     if prompt is not None and not is_chat_model(llm):
@@ -61,5 +66,5 @@ def tree_summarize(queries: List[str],
                                use_async=True)
     tasks = [summarizer.aget_response(query, content) for query, content in zip(queries, contents)]
     loop = asyncio.get_event_loop()
-    results = loop.run_until_complete(asyncio.gather(*tasks))
+    results = loop.run_until_complete(process_batch(tasks, batch_size=batch))
     return results

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/autorag/nodes/queryexpansion/base.py

@@ -35,6 +35,12 @@ def query_expansion_node(func):
         else:
             prompt = ""
 
+        # pop batch from kwargs
+        if "batch" in kwargs.keys():
+            batch = kwargs.pop("batch")
+        else:
+            batch = 16
+
         # set llm model for query expansion
         if llm_str in generator_models:
             llm = generator_models[llm_str](**kwargs)
@@ -43,7 +49,7 @@ def query_expansion_node(func):
             raise KeyError(f"llm_str {llm_str} does not exist.")
 
         # run query expansion function
-        expanded_queries = func(queries=queries, llm=llm, prompt=prompt)
+        expanded_queries = func(queries=queries, llm=llm, prompt=prompt, batch=batch)
 
         return expanded_queries
 

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/autorag/nodes/queryexpansion/hyde.py

@@ -4,13 +4,15 @@ from typing import List
 from llama_index.llms.llm import BaseLLM
 
 from autorag.nodes.queryexpansion.base import query_expansion_node
+from autorag.utils.util import process_batch
 
 hyde_prompt = "Please write a passage to answer the question"
 
 
 @query_expansion_node
 def hyde(queries: List[str], llm: BaseLLM,
-         prompt: str = hyde_prompt) -> List[List[str]]:
+         prompt: str = hyde_prompt,
+         batch: int = 16) -> List[List[str]]:
     """
     HyDE, which inspired by "Precise Zero-shot Dense Retrieval without Relevance Labels" (https://arxiv.org/pdf/2212.10496.pdf)
     LLM model creates hypothetical passage.
@@ -18,12 +20,14 @@ def hyde(queries: List[str], llm: BaseLLM,
     :param queries: List[str], queries to retrieve.
     :param llm: llm to use for hypothetical passage generation.
     :param prompt: prompt to use when generating hypothetical passage
+    :param batch: Batch size for llm.
+        Default is 16.
     :return: List[List[str]], List of hyde results.
     """
     # Run async query_decompose_pure function
     tasks = [hyde_pure(query, llm, prompt) for query in queries]
     loop = asyncio.get_event_loop()
-    results = loop.run_until_complete(asyncio.gather(*tasks))
+    results = loop.run_until_complete(process_batch(tasks, batch_size=batch))
     return results
 
 
@@ -32,5 +36,5 @@ async def hyde_pure(query: str, llm: BaseLLM,
     if prompt is "":
         prompt = hyde_prompt
     full_prompt = prompt + f"\nQuestion: {query}\nPassage:"
-    hyde_answer = llm.complete(full_prompt)
+    hyde_answer = await llm.acomplete(full_prompt)
     return [hyde_answer.text]

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/autorag/nodes/queryexpansion/query_decompose.py

@@ -4,6 +4,7 @@ from typing import List
 from llama_index.llms.llm import BaseLLM
 
 from autorag.nodes.queryexpansion.base import query_expansion_node
+from autorag.utils.util import process_batch
 
 decompose_prompt = """Decompose a question in self-contained sub-questions. Use \"The question needs no decomposition\" when no decomposition is needed.
 
@@ -54,19 +55,22 @@ decompose_prompt = """Decompose a question in self-contained sub-questions. Use
 
 @query_expansion_node
 def query_decompose(queries: List[str], llm: BaseLLM,
-                    prompt: str = decompose_prompt) -> List[List[str]]:
+                    prompt: str = decompose_prompt,
+                    batch: int = 16) -> List[List[str]]:
     """
     decompose query to little piece of questions.
     :param queries: List[str], queries to decompose.
     :param llm: BaseLLM, language model to use.
     :param prompt: str, prompt to use for query decomposition.
         default prompt comes from Visconde's StrategyQA few-shot prompt.
+    :param batch: int, batch size for llm.
+        Default is 16.
     :return: List[List[str]], list of decomposed query. Return input query if query is not decomposable.
     """
     # Run async query_decompose_pure function
     tasks = [query_decompose_pure(query, llm, prompt) for query in queries]
     loop = asyncio.get_event_loop()
-    results = loop.run_until_complete(asyncio.gather(*tasks))
+    results = loop.run_until_complete(process_batch(tasks, batch_size=batch))
     return results
 
 
@@ -83,7 +87,7 @@ async def query_decompose_pure(query: str, llm: BaseLLM,
     if prompt == "":
         prompt = decompose_prompt
     full_prompt = "prompt: " + prompt + "\n\n" "question: " + query
-    answer = llm.complete(full_prompt)
+    answer = await llm.acomplete(full_prompt)
     if answer.text == "the question needs no decomposition.":
         return [query]
     try:

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/autorag/nodes/retrieval/hybrid_cc.py

@@ -55,8 +55,9 @@ def hybrid_cc(
 
 def cc_pure(ids: Tuple, scores: Tuple, weights: Tuple, top_k: int) -> Tuple[
     List[str], List[float]]:
-    df = pd.concat([pd.Series(dict(zip(_id, score))) for _id, score in zip(ids, scores)], axis=1, join="inner")
+    df = pd.concat([pd.Series(dict(zip(_id, score))) for _id, score in zip(ids, scores)], axis=1)
     normalized_scores = (df - df.min()) / (df.max() - df.min())
+    normalized_scores = normalized_scores.fillna(0)
     normalized_scores['weighted_sum'] = normalized_scores.mul(weights).sum(axis=1)
     normalized_scores = normalized_scores.sort_values(by='weighted_sum', ascending=False)
     return normalized_scores.index.tolist()[:top_k], normalized_scores['weighted_sum'][:top_k].tolist()

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/autorag/nodes/retrieval/run.py

@@ -68,28 +68,37 @@ def run_retrieval_node(modules: List[Callable],
 
     # run retrieval modules except hybrid
     hybrid_module_names = ['hybrid_rrf', 'hybrid_cc']
-    non_hybrid_modules, non_hybrid_module_params = zip(*filter(lambda x: x[0].__name__ not in hybrid_module_names,
-                                                               zip(modules, module_params)))
     filename_first = 0
-    non_hybrid_results, non_hybrid_times, non_hybrid_summary_df = run_and_save(non_hybrid_modules,
-                                                                               non_hybrid_module_params, filename_first)
-    filename_first += len(non_hybrid_modules)
+    if any([module.__name__ not in hybrid_module_names for module in modules]):
+        non_hybrid_modules, non_hybrid_module_params = zip(*filter(lambda x: x[0].__name__ not in hybrid_module_names,
+                                                                   zip(modules, module_params)))
+        non_hybrid_results, non_hybrid_times, non_hybrid_summary_df = run_and_save(non_hybrid_modules,
+                                                                                   non_hybrid_module_params, filename_first)
+        filename_first += len(non_hybrid_modules)
+    else:
+        non_hybrid_results, non_hybrid_times, non_hybrid_summary_df = [], [], pd.DataFrame()
 
     if any([module.__name__ in hybrid_module_names for module in modules]):
         hybrid_modules, hybrid_module_params = zip(*filter(lambda x: x[0].__name__ in hybrid_module_names,
                                                            zip(modules, module_params)))
-        target_modules = list(map(lambda x: x.pop('target_modules'), hybrid_module_params))
-        target_filenames = list(map(lambda x: select_result_for_hybrid(save_dir, x), target_modules))
-        ids_scores = list(map(lambda x: get_ids_and_scores(save_dir, x), target_filenames))
-        target_module_params = list(map(lambda x: get_module_params(save_dir, x), target_filenames))
-        hybrid_module_params = list(map(lambda x: {**x[0], **x[1]}, zip(hybrid_module_params, ids_scores)))
-        real_hybrid_times = list(map(lambda filename: get_hybrid_execution_times(save_dir, filename), target_filenames))
-        hybrid_results, hybrid_times, hybrid_summary_df = run_and_save(hybrid_modules, hybrid_module_params,
-                                                                       filename_first)
-        filename_first += len(hybrid_modules)
-        hybrid_times = real_hybrid_times.copy()
-        hybrid_summary_df['execution_time'] = hybrid_times
-        hybrid_summary_df = edit_summary_df_params(hybrid_summary_df, target_modules, target_module_params)
+        if all(['target_module_params' in x for x in hybrid_module_params]):
+            # If target_module_params are already given, run hybrid retrieval directly
+            hybrid_results, hybrid_times, hybrid_summary_df = run_and_save(hybrid_modules, hybrid_module_params,
+                                                                           filename_first)
+            filename_first += len(hybrid_modules)
+        else:
+            target_modules = list(map(lambda x: x.pop('target_modules'), hybrid_module_params))
+            target_filenames = list(map(lambda x: select_result_for_hybrid(save_dir, x), target_modules))
+            ids_scores = list(map(lambda x: get_ids_and_scores(save_dir, x), target_filenames))
+            target_module_params = list(map(lambda x: get_module_params(save_dir, x), target_filenames))
+            hybrid_module_params = list(map(lambda x: {**x[0], **x[1]}, zip(hybrid_module_params, ids_scores)))
+            real_hybrid_times = list(map(lambda filename: get_hybrid_execution_times(save_dir, filename), target_filenames))
+            hybrid_results, hybrid_times, hybrid_summary_df = run_and_save(hybrid_modules, hybrid_module_params,
+                                                                           filename_first)
+            filename_first += len(hybrid_modules)
+            hybrid_times = real_hybrid_times.copy()
+            hybrid_summary_df['execution_time'] = hybrid_times
+            hybrid_summary_df = edit_summary_df_params(hybrid_summary_df, target_modules, target_module_params)
     else:
         hybrid_results, hybrid_times, hybrid_summary_df = [], [], pd.DataFrame()
 
@@ -161,11 +170,11 @@ def select_result_for_hybrid(node_dir: str, target_modules: Tuple) -> List[str]:
     return best_filenames
 
 
-def get_module_params(node_dir: str, filenames: List[str]) -> List[Dict]:
+def get_module_params(node_dir: str, filenames: List[str]) -> Tuple[Dict]:
     summary_df = load_summary_file(os.path.join(node_dir, "summary.csv"))
     best_results = summary_df[summary_df['filename'].isin(filenames)]
     module_params = best_results['module_params'].tolist()
-    return module_params
+    return tuple(module_params)
 
 
 def edit_summary_df_params(summary_df: pd.DataFrame, target_modules, target_module_params) -> pd.DataFrame:

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/autorag/nodes/retrieval/vectordb.py

@@ -7,11 +7,13 @@ from llama_index.embeddings import BaseEmbedding
 
 from autorag.nodes.retrieval.base import retrieval_node, evenly_distribute_passages
 from autorag.utils import validate_corpus_dataset
+from autorag.utils.util import process_batch
 
 
 @retrieval_node
 def vectordb(queries: List[List[str]], top_k: int, collection: chromadb.Collection,
-             embedding_model: BaseEmbedding) -> Tuple[List[List[str]], List[List[float]]]:
+             embedding_model: BaseEmbedding,
+             batch: int = 128) -> Tuple[List[List[str]], List[List[float]]]:
     """
     VectorDB retrieval function.
     You have to get chroma collection that is already ingested.
@@ -22,6 +24,9 @@ def vectordb(queries: List[List[str]], top_k: int, collection: chromadb.Collecti
     :param top_k: The number of passages to be retrieved.
     :param collection: A chroma collection instance that will be used to retrieve passages.
     :param embedding_model: An embedding model instance that will be used to embed queries.
+    :param batch: The number of queries to be processed in parallel.
+        This is used to prevent API error at the query embedding.
+        Default is 128.
 
     :return: The 2-d list contains a list of passage ids that retrieved from vectordb and 2-d list of its scores.
         It will be a length of queries. And each element has a length of top_k.
@@ -32,7 +37,7 @@ def vectordb(queries: List[List[str]], top_k: int, collection: chromadb.Collecti
     # run async vector_db_pure function
     tasks = [vectordb_pure(input_queries, top_k, collection, embedding_model) for input_queries in queries]
     loop = asyncio.get_event_loop()
-    results = loop.run_until_complete(asyncio.gather(*tasks))
+    results = loop.run_until_complete(process_batch(tasks, batch_size=batch))
     id_result = list(map(lambda x: x[0], results))
     score_result = list(map(lambda x: x[1], results))
     return id_result, score_result

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/autorag/utils/util.py

@@ -1,4 +1,5 @@
 import ast
+import asyncio
 import functools
 import itertools
 import os
@@ -200,3 +201,22 @@ def convert_string_to_tuple_in_dict(d):
             d[key] = ast.literal_eval(value)
 
     return d
+
+
+async def process_batch(tasks, batch_size: int = 64) -> List[Any]:
+    """
+    Processes tasks in batches asynchronously.
+
+    :param tasks: A list of no-argument functions or coroutines to be executed.
+    :param batch_size: The number of tasks to process in a single batch.
+        Default is 64.
+    :return: A list of results from the processed tasks.
+    """
+    results = []
+
+    for i in range(0, len(tasks), batch_size):
+        batch = tasks[i:i + batch_size]
+        batch_results = await asyncio.gather(*batch)
+        results.extend(batch_results)
+
+    return results

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/docs/source/conf.py

@@ -10,7 +10,7 @@ project = 'AutoRAG'
 copyright = '2024, Marker-Inc'
 author = 'Marker-Inc'
 
-with open('../../VERSION') as f:
+with open('../../autorag/VERSION') as f:
     version = f.read().strip()
 
 # -- General configuration ---------------------------------------------------

AutoRAG-0.0.4/docs/source/data_creation/tutorial.md

@@ -0,0 +1 @@
+# Start creating your own evaluation data

AutoRAG-0.0.4/docs/source/index.rst

@@ -0,0 +1,45 @@
+.. AutoRAG documentation master file, created by
+   sphinx-quickstart on Wed Jan 17 20:55:21 2024.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+AutoRAG documentation
+===================================
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Getting Started
+   :hidden:
+
+   install.md
+   tutorial.md
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Data Creation
+   :hidden:
+
+   data_creation/tutorial.md
+
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Optimization
+   :hidden:
+
+   optimization/optimization.md
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Available Modules
+   :hidden:
+
+   modules/index.md
+
+.. toctree::
+   :maxdepth: 1
+   :caption: API Reference
+   :hidden:
+
+   api_spec/modules
+

AutoRAG-0.0.4/docs/source/install.md

@@ -0,0 +1 @@
+# Installation

AutoRAG-0.0.4/docs/source/modules/index.md

@@ -0,0 +1 @@
+# Available Module List

AutoRAG-0.0.4/docs/source/optimization/optimization.md

@@ -0,0 +1,3 @@
+# How optimization works
+
+![Sundae](../resources/samsung_sundae.jpeg)

AutoRAG-0.0.4/docs/source/tutorial.md

@@ -0,0 +1 @@
+# Tutorial

{AutoRAG-0.0.2 → AutoRAG-0.0.4}/pyproject.toml

@@ -5,13 +5,13 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "AutoRAG"
 authors = [
-    {name = "Marker-Inc", email = "vkehfdl1@gmail.com"}
+    { name = "Marker-Inc", email = "vkehfdl1@gmail.com" }
 ]
 description = 'Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.'
 readme = "README.md"
 requires-python = ">=3.8"
 keywords = ['RAG', 'AutoRAG', 'autorag', 'rag-evaluation', 'evaluation', 'rag-auto', 'AutoML', 'AutoML-RAG']
-license = {file = "LICENSE"}
+license = { file = "LICENSE" }
 classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: Information Technology",
@@ -23,13 +23,15 @@ classifiers = [
     "Topic :: Software Development :: Libraries",
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
-urls = {Homepage = "https://github.com/Marker-Inc-Korea/AutoRAG"}
+urls = { Homepage = "https://github.com/Marker-Inc-Korea/AutoRAG" }
 dynamic = ["version", "dependencies"]
 
 [tool.setuptools.dynamic]
-version = {file = ["VERSION"]}
-dependencies = {file = ["requirements.txt"]}
+version = { file = ["autorag/VERSION"] }
+dependencies = { file = ["requirements.txt"] }
 
+[tool.setuptools]
+include-package-data = true
 
 [tool.setuptools.packages.find]
 where = ["."]

AutoRAG-0.0.4/sample_dataset/README.md

@@ -0,0 +1,25 @@
+# sample_dataset handling
+
+The sample_dataset folder does not includes a `qa.parquet`, `corpus.parquet` file that is significantly large and cannot be uploaded directly to Git due to size limitations. 
+
+To prepare and use datasets available in the sample_dataset folder, specifically `triviaqa`, `msmarco` and `eli5`, you can follow the outlined methods below.
+
+## Usage
+
+ The example provided uses `triviaqa`, but the same approach applies to `msmarco` and `eli5`.
+
+### 1. Run with a specified save path
+To execute the Python script from the terminal and save the dataset to a specified path, use the command:
+
+```bash
+python ./sample_dataset/triviaqa/load_triviaqa_dataset.py --save_path /path/to/save/dataset
+```
+This runs the `load_triviaqa_dataset.py` script located in the `./sample_dataset/triviaqa/` directory,
+using the `--save_path` argument to specify the dataset's save location.
+
+### 2. Run without specifying a save path
+If you run the script without the `--save_path` argument, the dataset will be saved to a default location, which is the directory containing the `load_triviaqa_dataset.py` file, essentially `./sample_dataset/triviaqa/`:
+```bash
+python ./sample_dataset/triviaqa/load_triviaqa_dataset.py
+```
+This behavior allows for a straightforward execution without needing to specify a path, making it convenient for quick tests or when working directly within the target directory.