AutoRAG 0.3.6__tar.gz → 0.3.7__tar.gz

{autorag-0.3.6 → autorag-0.3.7}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.3.6
+Version: 0.3.7
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -240,6 +240,8 @@ Requires-Dist: aiohttp
 Requires-Dist: voyageai
 Requires-Dist: mixedbread-ai
 Requires-Dist: llama-index-llms-bedrock
+Requires-Dist: scikit-learn
+Requires-Dist: pymilvus
 Requires-Dist: quart
 Requires-Dist: llama-index>=0.11.0
 Requires-Dist: llama-index-core>=0.11.0
@@ -304,6 +306,14 @@ RAG AutoML tool for automatically finding an optimal RAG pipeline for your data.
 
 ![Thumbnail](https://github.com/user-attachments/assets/6bab243d-a4b3-431a-8ac0-fe17336ab4de)
 
+![Discord](https://img.shields.io/discord/1204010535272587264) ![PyPI - Downloads](https://img.shields.io/pypi/dm/AutoRAG)
+[![LinkedIn](https://img.shields.io/badge/LinkedIn-Connect-blue?style=flat-square&logo=linkedin)](https://www.linkedin.com/company/104375108/admin/dashboard/)
+![X (formerly Twitter) Follow](https://img.shields.io/twitter/follow/AutoRAG_HQ)
+[![Hugging Face](https://img.shields.io/badge/Hugging%20Face-Follow-orange?style=flat-square&logo=huggingface)](https://huggingface.co/AutoRAG)
+[![Static Badge](https://img.shields.io/badge/Roadmap-5D3FD3)](https://github.com/orgs/Auto-RAG/projects/1/views/2)
+
+
+
 There are many RAG pipelines and modules out there,
 but you don’t know what pipeline is great for “your own data” and "your own use-case."
 Making and evaluating all RAG modules is very time-consuming and hard to do.
@@ -608,7 +618,7 @@ node_lines:
       top_k: 3
       modules:
         - module_type: vectordb
-          embedding_model: openai
+          vectordb: default
         - module_type: bm25
         - module_type: hybrid_rrf
           weight_range: (4,80)
@@ -753,7 +763,7 @@ at [here](https://edai.notion.site/Supporting-metrics-867d71caefd7401c9264dd91ba
 
 ## ☎️ FaQ
 
-🛣️ [Support plans & Roadmap](https://edai.notion.site/Support-plans-Roadmap-02ca7c97376340c393885855e2d99630?pvs=4)
+🛣️ [Roadmap](https://github.com/orgs/Auto-RAG/projects/1/views/2)
 
 💻 [Hardware Specs](https://edai.notion.site/Hardware-specs-28cefcf2a26246ffadc91e2f3dc3d61c?pvs=4)
 

{autorag-0.3.6 → autorag-0.3.7}/AutoRAG.egg-info/SOURCES.txt

@@ -182,6 +182,10 @@ autorag/schema/node.py
 autorag/utils/__init__.py
 autorag/utils/preprocess.py
 autorag/utils/util.py
+autorag/vectordb/__init__.py
+autorag/vectordb/base.py
+autorag/vectordb/chroma.py
+autorag/vectordb/milvus.py
 docker/base/Dockerfile
 docker/gpu/Dockerfile
 docs/Makefile
@@ -256,6 +260,7 @@ docs/source/api_spec/autorag.nodes.rst
 docs/source/api_spec/autorag.rst
 docs/source/api_spec/autorag.schema.rst
 docs/source/api_spec/autorag.utils.rst
+docs/source/api_spec/autorag.vectordb.rst
 docs/source/api_spec/modules.rst
 docs/source/data_creation/data_creation.md
 docs/source/data_creation/data_format.md
@@ -333,6 +338,9 @@ docs/source/optimization/folder_structure.md
 docs/source/optimization/optimization.md
 docs/source/optimization/strategies.md
 docs/source/roadmap/modular_rag.md
+docs/source/vectordb/chroma.md
+docs/source/vectordb/milvus.md
+docs/source/vectordb/vectordb.md
 projects/tutorial_1/config.yaml
 sample_config/chunk/chunk_full.yaml
 sample_config/chunk/chunk_ko.yaml
@@ -340,6 +348,7 @@ sample_config/chunk/simple_chunk.yaml
 sample_config/parse/parse_full.yaml
 sample_config/parse/parse_hybird.yaml
 sample_config/parse/parse_ko.yaml
+sample_config/parse/parse_multimodal.yaml
 sample_config/parse/parse_ocr.yaml
 sample_config/parse/simple_parse.yaml
 sample_config/rag/extracted_sample.yaml
@@ -491,6 +500,9 @@ tests/autorag/schema/test_module_schema.py
 tests/autorag/schema/test_node_schema.py
 tests/autorag/utils/test_preprocess.py
 tests/autorag/utils/test_util.py
+tests/autorag/vectordb/test_base.py
+tests/autorag/vectordb/test_chroma.py
+tests/autorag/vectordb/test_milvus.py
 tests/resources/README.md
 tests/resources/corpus_data_sample.parquet
 tests/resources/full.yaml
@@ -499,6 +511,7 @@ tests/resources/qa_test_data_sample.parquet
 tests/resources/sample_contents_nqa.csv
 tests/resources/simple.yaml
 tests/resources/simple_chunk.yaml
+tests/resources/simple_milvus.yaml
 tests/resources/simple_mock.yaml
 tests/resources/simple_parse.yaml
 tests/resources/simple_with_llm.yaml
@@ -601,6 +614,7 @@ tests/resources/result_project/3/config.yaml
 tests/resources/result_project/data/corpus.parquet
 tests/resources/result_project/data/qa.parquet
 tests/resources/result_project/resources/bm25_porter_stemmer.pkl
+tests/resources/result_project/resources/vectordb.yaml
 tests/resources/result_project/resources/chroma/chroma.sqlite3
 tests/resources/result_project/resources/chroma/6595d304-5270-4d9f-a267-c191366804ce/data_level0.bin
 tests/resources/result_project/resources/chroma/6595d304-5270-4d9f-a267-c191366804ce/header.bin

{autorag-0.3.6 → autorag-0.3.7}/AutoRAG.egg-info/requires.txt

@@ -19,6 +19,8 @@ aiohttp
 voyageai
 mixedbread-ai
 llama-index-llms-bedrock
+scikit-learn
+pymilvus
 quart
 llama-index>=0.11.0
 llama-index-core>=0.11.0

{autorag-0.3.6 → autorag-0.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.3.6
+Version: 0.3.7
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -240,6 +240,8 @@ Requires-Dist: aiohttp
 Requires-Dist: voyageai
 Requires-Dist: mixedbread-ai
 Requires-Dist: llama-index-llms-bedrock
+Requires-Dist: scikit-learn
+Requires-Dist: pymilvus
 Requires-Dist: quart
 Requires-Dist: llama-index>=0.11.0
 Requires-Dist: llama-index-core>=0.11.0
@@ -304,6 +306,14 @@ RAG AutoML tool for automatically finding an optimal RAG pipeline for your data.
 
 ![Thumbnail](https://github.com/user-attachments/assets/6bab243d-a4b3-431a-8ac0-fe17336ab4de)
 
+![Discord](https://img.shields.io/discord/1204010535272587264) ![PyPI - Downloads](https://img.shields.io/pypi/dm/AutoRAG)
+[![LinkedIn](https://img.shields.io/badge/LinkedIn-Connect-blue?style=flat-square&logo=linkedin)](https://www.linkedin.com/company/104375108/admin/dashboard/)
+![X (formerly Twitter) Follow](https://img.shields.io/twitter/follow/AutoRAG_HQ)
+[![Hugging Face](https://img.shields.io/badge/Hugging%20Face-Follow-orange?style=flat-square&logo=huggingface)](https://huggingface.co/AutoRAG)
+[![Static Badge](https://img.shields.io/badge/Roadmap-5D3FD3)](https://github.com/orgs/Auto-RAG/projects/1/views/2)
+
+
+
 There are many RAG pipelines and modules out there,
 but you don’t know what pipeline is great for “your own data” and "your own use-case."
 Making and evaluating all RAG modules is very time-consuming and hard to do.
@@ -608,7 +618,7 @@ node_lines:
       top_k: 3
       modules:
         - module_type: vectordb
-          embedding_model: openai
+          vectordb: default
         - module_type: bm25
         - module_type: hybrid_rrf
           weight_range: (4,80)
@@ -753,7 +763,7 @@ at [here](https://edai.notion.site/Supporting-metrics-867d71caefd7401c9264dd91ba
 
 ## ☎️ FaQ
 
-🛣️ [Support plans & Roadmap](https://edai.notion.site/Support-plans-Roadmap-02ca7c97376340c393885855e2d99630?pvs=4)
+🛣️ [Roadmap](https://github.com/orgs/Auto-RAG/projects/1/views/2)
 
 💻 [Hardware Specs](https://edai.notion.site/Hardware-specs-28cefcf2a26246ffadc91e2f3dc3d61c?pvs=4)
 

{autorag-0.3.6 → autorag-0.3.7}/README.md

@@ -4,6 +4,14 @@ RAG AutoML tool for automatically finding an optimal RAG pipeline for your data.
 
 ![Thumbnail](https://github.com/user-attachments/assets/6bab243d-a4b3-431a-8ac0-fe17336ab4de)
 
+![Discord](https://img.shields.io/discord/1204010535272587264) ![PyPI - Downloads](https://img.shields.io/pypi/dm/AutoRAG)
+[![LinkedIn](https://img.shields.io/badge/LinkedIn-Connect-blue?style=flat-square&logo=linkedin)](https://www.linkedin.com/company/104375108/admin/dashboard/)
+![X (formerly Twitter) Follow](https://img.shields.io/twitter/follow/AutoRAG_HQ)
+[![Hugging Face](https://img.shields.io/badge/Hugging%20Face-Follow-orange?style=flat-square&logo=huggingface)](https://huggingface.co/AutoRAG)
+[![Static Badge](https://img.shields.io/badge/Roadmap-5D3FD3)](https://github.com/orgs/Auto-RAG/projects/1/views/2)
+
+
+
 There are many RAG pipelines and modules out there,
 but you don’t know what pipeline is great for “your own data” and "your own use-case."
 Making and evaluating all RAG modules is very time-consuming and hard to do.
@@ -308,7 +316,7 @@ node_lines:
       top_k: 3
       modules:
         - module_type: vectordb
-          embedding_model: openai
+          vectordb: default
         - module_type: bm25
         - module_type: hybrid_rrf
           weight_range: (4,80)
@@ -453,7 +461,7 @@ at [here](https://edai.notion.site/Supporting-metrics-867d71caefd7401c9264dd91ba
 
 ## ☎️ FaQ
 
-🛣️ [Support plans & Roadmap](https://edai.notion.site/Support-plans-Roadmap-02ca7c97376340c393885855e2d99630?pvs=4)
+🛣️ [Roadmap](https://github.com/orgs/Auto-RAG/projects/1/views/2)
 
 💻 [Hardware Specs](https://edai.notion.site/Hardware-specs-28cefcf2a26246ffadc91e2f3dc3d61c?pvs=4)
 

autorag-0.3.7/autorag/VERSION

@@ -0,0 +1 @@
+0.3.7

{autorag-0.3.6 → autorag-0.3.7}/autorag/cli.py

@@ -159,7 +159,7 @@ def extract_best_config(trial_path: str, output_path: str):
 def restart_evaluate(trial_path):
 	if not os.path.exists(trial_path):
 		raise ValueError(f"trial_path {trial_path} does not exist.")
-	project_dir = pathlib.PurePath(trial_path).parent
+	project_dir = str(pathlib.PurePath(trial_path).parent)
 	qa_data_path = os.path.join(project_dir, "data", "qa.parquet")
 	corpus_data_path = os.path.join(project_dir, "data", "corpus.parquet")
 	evaluator = Evaluator(qa_data_path, corpus_data_path, project_dir)

{autorag-0.3.6 → autorag-0.3.7}/autorag/data/legacy/qacreation/base.py

@@ -144,6 +144,7 @@ def make_qa_with_existing_qa(
 	:param kwargs: The keyword arguments for qa_creation_func.
 	:return: QA dataset dataframe.
 	"""
+	raise DeprecationWarning("This function is deprecated.")
 	assert (
 		"query" in existing_query_df.columns
 	), "existing_query_df must have 'query' column."

autorag-0.3.7/autorag/data/parse/llamaparse.py

@@ -0,0 +1,126 @@
+import os
+from typing import List, Tuple
+from itertools import chain
+
+from llama_parse import LlamaParse
+
+from autorag.data.parse.base import parser_node
+from autorag.utils.util import process_batch, get_event_loop
+
+
+@parser_node
+def llama_parse(
+	data_path_list: List[str],
+	batch: int = 8,
+	use_vendor_multimodal_model: bool = False,
+	vendor_multimodal_model_name: str = "openai-gpt4o",
+	use_own_key: bool = False,
+	vendor_multimodal_api_key: str = None,
+	**kwargs,
+) -> Tuple[List[str], List[str], List[int]]:
+	"""
+	Parse documents to use llama_parse.
+	LLAMA_CLOUD_API_KEY environment variable should be set.
+	You can get the key from https://cloud.llamaindex.ai/api-key
+
+	:param data_path_list: The list of data paths to parse.
+	:param batch: The batch size for parse documents. Default is 8.
+	:param use_vendor_multimodal_model: Whether to use the vendor multimodal model. Default is False.
+	:param vendor_multimodal_model_name: The name of the vendor multimodal model. Default is "openai-gpt4o".
+	:param use_own_key: Whether to use the own API key. Default is False.
+	:param vendor_multimodal_api_key: The API key for the vendor multimodal model.
+	:param kwargs: The extra parameters for creating the llama_parse instance.
+	:return: tuple of lists containing the parsed texts, path and pages.
+	"""
+	if use_vendor_multimodal_model:
+		kwargs = _add_multimodal_params(
+			kwargs,
+			use_vendor_multimodal_model,
+			vendor_multimodal_model_name,
+			use_own_key,
+			vendor_multimodal_api_key,
+		)
+
+	parse_instance = LlamaParse(**kwargs)
+
+	tasks = [
+		llama_parse_pure(data_path, parse_instance) for data_path in data_path_list
+	]
+	loop = get_event_loop()
+	results = loop.run_until_complete(process_batch(tasks, batch))
+
+	del parse_instance
+
+	texts, path, pages = (list(chain.from_iterable(item)) for item in zip(*results))
+
+	return texts, path, pages
+
+
+async def llama_parse_pure(
+	data_path: str, parse_instance
+) -> Tuple[List[str], List[str], List[int]]:
+	documents = await parse_instance.aload_data(data_path)
+
+	texts = list(map(lambda x: x.text, documents))
+	path = [data_path] * len(texts)
+	pages = list(range(1, len(documents) + 1))
+
+	return texts, path, pages
+
+
+def _add_multimodal_params(
+	kwargs,
+	use_vendor_multimodal_model,
+	vendor_multimodal_model_name,
+	use_own_key,
+	vendor_multimodal_api_key,
+) -> dict:
+	kwargs["use_vendor_multimodal_model"] = use_vendor_multimodal_model
+	kwargs["vendor_multimodal_model_name"] = vendor_multimodal_model_name
+
+	def set_multimodal_api_key(
+		multimodal_model_name: str = "openai-gpt4o", _api_key: str = None
+	) -> str:
+		if multimodal_model_name in ["openai-gpt4o", "openai-gpt-4o-mini"]:
+			_api_key = (
+				os.getenv("OPENAI_API_KEY", None) if _api_key is None else _api_key
+			)
+			if _api_key is None:
+				raise KeyError(
+					"Please set the OPENAI_API_KEY in the environment variable OPENAI_API_KEY "
+					"or directly set it on the config YAML file."
+				)
+		elif multimodal_model_name in ["anthropic-sonnet-3.5"]:
+			_api_key = (
+				os.getenv("ANTHROPIC_API_KEY", None) if _api_key is None else _api_key
+			)
+			if _api_key is None:
+				raise KeyError(
+					"Please set the ANTHROPIC_API_KEY in the environment variable ANTHROPIC_API_KEY "
+					"or directly set it on the config YAML file."
+				)
+		elif multimodal_model_name in ["gemini-1.5-flash", "gemini-1.5-pro"]:
+			_api_key = (
+				os.getenv("GEMINI_API_KEY", None) if _api_key is None else _api_key
+			)
+			if _api_key is None:
+				raise KeyError(
+					"Please set the GEMINI_API_KEY in the environment variable GEMINI_API_KEY "
+					"or directly set it on the config YAML file."
+				)
+		elif multimodal_model_name in ["custom-azure-model"]:
+			raise NotImplementedError(
+				"Custom Azure multimodal model is not supported yet."
+			)
+		else:
+			raise ValueError("Invalid multimodal model name.")
+
+		return _api_key
+
+	if use_own_key:
+		api_key = set_multimodal_api_key(
+			vendor_multimodal_model_name, vendor_multimodal_api_key
+		)
+		kwargs["vendor_multimodal_api_key"] = api_key
+
+	return kwargs

{autorag-0.3.6 → autorag-0.3.7}/autorag/deploy/base.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import pathlib
 import uuid
 from copy import deepcopy
 from typing import Optional, Dict, List
@@ -8,8 +9,7 @@ import pandas as pd
 import yaml
 
 from autorag.support import get_support_modules
-from autorag.utils.util import load_summary_file
-
+from autorag.utils.util import load_summary_file, load_yaml_config
 
 logger = logging.getLogger("AutoRAG")
 
@@ -94,13 +94,13 @@ def summary_df_to_yaml(summary_df: pd.DataFrame, config_dict: Dict) -> Dict:
 
 def extract_best_config(trial_path: str, output_path: Optional[str] = None) -> Dict:
 	"""
-	Extract the optimal pipeline from evaluated trial.
+	Extract the optimal pipeline from the evaluated trial.
 
 	:param trial_path: The path to the trial directory that you want to extract the pipeline from.
 	    Must already be evaluated.
 	:param output_path: Output path that pipeline yaml file will be saved.
 	    Must be .yaml or .yml file.
-	    If None, it does not save yaml file and just return dict values.
+	    If None, it does not save YAML file and just returns dict values.
 	    Default is None.
 	:return: The dictionary of the extracted pipeline.
 	"""
@@ -114,18 +114,42 @@ def extract_best_config(trial_path: str, output_path: Optional[str] = None) -> D
 	with open(config_yaml_path, "r") as f:
 		config_dict = yaml.safe_load(f)
 	yaml_dict = summary_df_to_yaml(trial_summary_df, config_dict)
+	yaml_dict["vectordb"] = extract_vectordb_config(trial_path)
 	if output_path is not None:
 		with open(output_path, "w") as f:
 			yaml.safe_dump(yaml_dict, f)
 	return yaml_dict
 
 
+def extract_vectordb_config(trial_path: str) -> List[Dict]:
+	# get vectordb.yaml file
+	project_dir = pathlib.PurePath(os.path.realpath(trial_path)).parent
+	vectordb_config_path = os.path.join(project_dir, "resources", "vectordb.yaml")
+	if not os.path.exists(vectordb_config_path):
+		raise ValueError(f"vectordb.yaml does not exist in {vectordb_config_path}.")
+	with open(vectordb_config_path, "r") as f:
+		vectordb_dict = yaml.safe_load(f)
+	result = vectordb_dict.get("vectordb", [])
+	if len(result) != 0:
+		return result
+	# return default setting of chroma
+	return [
+		{
+			"name": "default",
+			"db_type": "chroma",
+			"client_type": "persistent",
+			"embedding_model": "openai",
+			"collection_name": "openai",
+			"path": os.path.join(project_dir, "resources", "chroma"),
+		}
+	]
+
+
 class BaseRunner:
 	def __init__(self, config: Dict, project_dir: Optional[str] = None):
 		self.config = config
 		project_dir = os.getcwd() if project_dir is None else project_dir
-		# self.app = Flask(__name__)
-		# self.__add_api_route()
+		os.environ["PROJECT_DIR"] = project_dir
 
 		# init modules
 		node_lines = deepcopy(self.config["node_lines"])
@@ -159,12 +183,7 @@ class BaseRunner:
 			Default is the current directory.
 		:return: Initialized Runner.
 		"""
-		with open(yaml_path, "r") as f:
-			try:
-				config = yaml.safe_load(f)
-			except yaml.YAMLError as exc:
-				logger.error(exc)
-				raise exc
+		config = load_yaml_config(yaml_path)
 		return cls(config, project_dir=project_dir)
 
 	@classmethod

{autorag-0.3.6 → autorag-0.3.7}/autorag/evaluation/metric/util.py

@@ -15,6 +15,14 @@ def calculate_cosine_similarity(a, b):
 	return similarity
 
 
+def calculate_l2_distance(a, b):
+	return np.linalg.norm(a - b)
+
+
+def calculate_inner_product(a, b):
+	return np.dot(a, b)
+
+
 def autorag_metric(fields_to_check: List[str]):
 	def decorator_autorag_metric(func):
 		@functools.wraps(func)