AutoRAG 0.2.13__tar.gz → 0.2.15__tar.gz

autorag-0.2.15/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,35 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: "[BUG] "
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Full Error log**
+If applicable, add full error log to help explain your problem.
+
+**Code that bug is happened**
+If applicable, add the code that bug is happened.
+(Especially, your AutoRAG YAML file or python codes that you wrote)
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. Windows, Linux, MacOS]
+ - Python version [e.g. 3.10]
+
+**Additional context**
+Add any other context about the problem here.

autorag-0.2.15/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: "[Feature Request]"
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

autorag-0.2.15/.pre-commit-config.yaml

@@ -0,0 +1,14 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0  # Use the ref you want to point at
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.6.1
+    hooks:
+      - id: ruff
+        types_or: [ python, pyi ]
+        args: [ "--fix" ]
+      - id: ruff-format
+        types_or: [ python, pyi ]

{autorag-0.2.13 → autorag-0.2.15}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.13
+Version: 0.2.15
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -221,13 +221,12 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy<2.0.0
-Requires-Dist: pandas
+Requires-Dist: pandas>=2.1.0
 Requires-Dist: tqdm
 Requires-Dist: tiktoken>=0.7.0
 Requires-Dist: openai>=1.0.0
 Requires-Dist: rank_bm25
 Requires-Dist: transformers
-Requires-Dist: swifter
 Requires-Dist: pyyaml
 Requires-Dist: pyarrow
 Requires-Dist: fastparquet
@@ -242,7 +241,7 @@ Requires-Dist: uvicorn
 Requires-Dist: torch
 Requires-Dist: sentencepiece
 Requires-Dist: guidance
-Requires-Dist: cohere<=5.6.2
+Requires-Dist: cohere>=5.8.0
 Requires-Dist: tokenlog>=0.0.2
 Requires-Dist: aiohttp
 Requires-Dist: bert_score
@@ -250,8 +249,9 @@ Requires-Dist: sentence-transformers
 Requires-Dist: FlagEmbedding
 Requires-Dist: ragas
 Requires-Dist: llmlingua
-Requires-Dist: llama-index>=0.10.1
-Requires-Dist: llama-index-core>=0.10.1
+Requires-Dist: peft
+Requires-Dist: llama-index>=0.11.0
+Requires-Dist: llama-index-core>=0.11.0
 Requires-Dist: llama-index-readers-file
 Requires-Dist: llama-index-embeddings-openai
 Requires-Dist: llama-index-embeddings-huggingface
@@ -261,6 +261,7 @@ Requires-Dist: llama-index-llms-openai-like
 Requires-Dist: llama-index-llms-ollama
 Requires-Dist: llama-index-retrievers-bm25
 Requires-Dist: streamlit
+Requires-Dist: gradio
 Requires-Dist: langchain-core>=0.1.6
 Requires-Dist: panel
 Requires-Dist: seaborn
@@ -270,8 +271,12 @@ Requires-Dist: ipywidgets_bokeh
 Provides-Extra: ko
 Requires-Dist: kiwipiepy; extra == "ko"
 Requires-Dist: konlpy; extra == "ko"
+Provides-Extra: dev
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
 Provides-Extra: all
 Requires-Dist: AutoRAG[ko]; extra == "all"
+Requires-Dist: AutoRAG[dev]; extra == "all"
 
 # AutoRAG
 

{autorag-0.2.13 → autorag-0.2.15}/AutoRAG.egg-info/SOURCES.txt

@@ -1,4 +1,5 @@
 .gitignore
+.pre-commit-config.yaml
 CODE_OF_CONDUCT.md
 CONTRIBUTING.md
 LICENSE
@@ -7,6 +8,8 @@ pyproject.toml
 requirements.txt
 .github/FUNDING.yml
 .github/dependabot.yml
+.github/ISSUE_TEMPLATE/bug_report.md
+.github/ISSUE_TEMPLATE/feature_request.md
 .github/workflows/publish.yml
 .github/workflows/sphinx.yml
 .github/workflows/test.yml
@@ -160,6 +163,7 @@ docs/source/_static/trial_folder.png
 docs/source/_static/trial_json.png
 docs/source/_static/trial_summary.png
 docs/source/_static/web_interface.png
+docs/source/_static/web_interface_gradio.png
 docs/source/_static/roadmap/RAG_paradigms.png
 docs/source/_static/roadmap/advanced_RAG.png
 docs/source/_static/roadmap/cycle.png

{autorag-0.2.13 → autorag-0.2.15}/AutoRAG.egg-info/requires.txt

@@ -1,11 +1,10 @@
 numpy<2.0.0
-pandas
+pandas>=2.1.0
 tqdm
 tiktoken>=0.7.0
 openai>=1.0.0
 rank_bm25
 transformers
-swifter
 pyyaml
 pyarrow
 fastparquet
@@ -20,7 +19,7 @@ uvicorn
 torch
 sentencepiece
 guidance
-cohere<=5.6.2
+cohere>=5.8.0
 tokenlog>=0.0.2
 aiohttp
 bert_score
@@ -28,8 +27,9 @@ sentence-transformers
 FlagEmbedding
 ragas
 llmlingua
-llama-index>=0.10.1
-llama-index-core>=0.10.1
+peft
+llama-index>=0.11.0
+llama-index-core>=0.11.0
 llama-index-readers-file
 llama-index-embeddings-openai
 llama-index-embeddings-huggingface
@@ -39,6 +39,7 @@ llama-index-llms-openai-like
 llama-index-llms-ollama
 llama-index-retrievers-bm25
 streamlit
+gradio
 langchain-core>=0.1.6
 panel
 seaborn
@@ -48,6 +49,11 @@ ipywidgets_bokeh
 
 [all]
 AutoRAG[ko]
+AutoRAG[dev]
+
+[dev]
+ruff
+pre-commit
 
 [ko]
 kiwipiepy

autorag-0.2.15/CONTRIBUTING.md

@@ -0,0 +1,106 @@
+# Getting Started
+
+Thank you so much for your consideration of contributing to AutoRAG!
+Okay Let's get started!
+
+## Quickstart
+
+1. Fork the repo and clone your fork
+2. Create a new virtual environment and activate it
+3. Install the dependencies and package (`pip install -e '.[all]'`)
+4. Install docs and test dependencies (`pip install -r docs/requirements.txt` and `pip install -r tests/requirements.txt`)
+
+Also, join our Discord at [here](https://discord.gg/P4DYXfmSAs) and discuss about your contribution with the community.
+
+## Contribution Guidelines
+
+### What should I work on?
+
+1. Add new modules or nodes to the AutoRAG.
+2. Fix bugs
+3. Add new feature to the AutoRAG
+4. Add new sample config YAML files
+5. Add new sample dataset
+6. Improve code quality & documentation
+7. Report a bug or suggest new feature
+
+It can be great you search 'good first issue' tag on github issue tab.
+
+# Development Guidelines
+
+## Repo Structure
+
+The `AutoRAG` repo have a standard python library structure.
+
+You can find the core part of the library in `autorag` folder.
+You can find the test codes of the `AutoRAG` at the `tests` folder.
+Plus, you can find the [AutoRAG docs](https://docs.auto-rag.com) at the `docs` folder.
+
+For resources, you can find sample config YAML files at `sample_config` folder.
+Plus, you can find sample dataset download script from `sample_dataset` folder.
+
+## Setting up environment
+
+As a python package, AutoRAG tested primarily in Python versions >= 3.9.
+Here's a guide to set environment.
+
+1. Fork [AutoRAG Github repo](https://github.com/Marker-Inc-Korea/AutoRAG) and clone it to your local machine. (New to GitHub / git? Here's [how](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo).)
+2. Open your cloned repository to your IDE or open your project at the terminal through `cd`.
+3. Install pre-commit hooks by running `pre-commit install` in the terminal.
+4. Make a new virtual environment (Highly Recommended)
+5. Install AutoRAG as development version. `pip install -e '.[all]'`
+6. Install test dependencies `pip install -r tests/requirements.txt`
+7. Write `pytest.ini` file and add env variable for running tests.
+```ini
+[pytest]
+env =
+    OPENAI_API_KEY=sk-xxxx
+
+log_cli=true
+log_cli_level=INFO
+```
+8. Install docs dependencies `pip install -r docs/requirements.txt`
+
+## Validating your change
+
+Let's make sure to `format/lint` our change.
+For bigger changes, let's also make sure to `test` our change.
+
+### Formatting/Linting
+
+We use `ruff` to linting and formatting our code.
+If you have installed pre-commit hooks in this repo, they should have taken care of the formatting and linting automatically.
+(Use `pre-commit install`)
+
+If you want to do it manually, you can lint and format with ruff.
+
+```bash
+ruff check --fix
+ruff format
+```
+
+### Testing
+
+If you modified or added new code logic, you have to create tests for them, too.
+- In almost all cases, add unit tests.
+- If your new features uses heavy GPU models or outside API connections, you must use mock and pytest fixtures.
+- The added test case must not leave any extra files.
+- You must cover all cases of features or adjust that you made.
+- If you don't know well about `pytest`, `pytest-env` and pytest fixtures and mock, please learn about it.
+- Lastly, if the test you made must need CUDA-enabled device, you can mark the test with `@pytest.mark.skipif(is_github_action())` so we can skip it on github actions.
+
+Plus, your test will run as github actions when you try to merge it to the main branch.
+
+#### Warning about OPENAI_API_KEY
+
+When executing test at github actions, it will be failed due to the OpenAI API key.
+Don't worry, we will run the test locally and merge your code if there is no problem.
+
+## Making a Pull Request
+
+You can make a pull request to the main branch of the AutoRAG repo.
+At the fork repository, you will make a PR to AutoRAG repo.
+You can assign the issue with `close #issue_number` to automatically close the issue when the PR is merged.
+
+After that, please assign `vkehfdl1` or `bwook00` as a reviewer.
+For merge, you will have to get at least one approval from the reviewer.

{autorag-0.2.13 → autorag-0.2.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.13
+Version: 0.2.15
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -221,13 +221,12 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: numpy<2.0.0
-Requires-Dist: pandas
+Requires-Dist: pandas>=2.1.0
 Requires-Dist: tqdm
 Requires-Dist: tiktoken>=0.7.0
 Requires-Dist: openai>=1.0.0
 Requires-Dist: rank_bm25
 Requires-Dist: transformers
-Requires-Dist: swifter
 Requires-Dist: pyyaml
 Requires-Dist: pyarrow
 Requires-Dist: fastparquet
@@ -242,7 +241,7 @@ Requires-Dist: uvicorn
 Requires-Dist: torch
 Requires-Dist: sentencepiece
 Requires-Dist: guidance
-Requires-Dist: cohere<=5.6.2
+Requires-Dist: cohere>=5.8.0
 Requires-Dist: tokenlog>=0.0.2
 Requires-Dist: aiohttp
 Requires-Dist: bert_score
@@ -250,8 +249,9 @@ Requires-Dist: sentence-transformers
 Requires-Dist: FlagEmbedding
 Requires-Dist: ragas
 Requires-Dist: llmlingua
-Requires-Dist: llama-index>=0.10.1
-Requires-Dist: llama-index-core>=0.10.1
+Requires-Dist: peft
+Requires-Dist: llama-index>=0.11.0
+Requires-Dist: llama-index-core>=0.11.0
 Requires-Dist: llama-index-readers-file
 Requires-Dist: llama-index-embeddings-openai
 Requires-Dist: llama-index-embeddings-huggingface
@@ -261,6 +261,7 @@ Requires-Dist: llama-index-llms-openai-like
 Requires-Dist: llama-index-llms-ollama
 Requires-Dist: llama-index-retrievers-bm25
 Requires-Dist: streamlit
+Requires-Dist: gradio
 Requires-Dist: langchain-core>=0.1.6
 Requires-Dist: panel
 Requires-Dist: seaborn
@@ -270,8 +271,12 @@ Requires-Dist: ipywidgets_bokeh
 Provides-Extra: ko
 Requires-Dist: kiwipiepy; extra == "ko"
 Requires-Dist: konlpy; extra == "ko"
+Provides-Extra: dev
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
 Provides-Extra: all
 Requires-Dist: AutoRAG[ko]; extra == "all"
+Requires-Dist: AutoRAG[dev]; extra == "all"
 
 # AutoRAG
 

autorag-0.2.15/autorag/VERSION

@@ -0,0 +1 @@
+0.2.15

autorag-0.2.15/autorag/__init__.py

@@ -0,0 +1,88 @@
+import logging
+import logging.config
+import os
+import sys
+
+import transformers
+from llama_index.core.llms.mock import MockLLM
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.embeddings.openai import OpenAIEmbedding
+from llama_index.embeddings.openai import OpenAIEmbeddingModelType
+from llama_index.llms.huggingface import HuggingFaceLLM
+from llama_index.llms.ollama import Ollama
+from llama_index.llms.openai import OpenAI
+from llama_index.llms.openai_like import OpenAILike
+from rich.logging import RichHandler
+
+version_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "VERSION")
+
+with open(version_path, "r") as f:
+	__version__ = f.read().strip()
+
+
+class LazyInit:
+	def __init__(self, factory, *args, **kwargs):
+		self._factory = factory
+		self._args = args
+		self._kwargs = kwargs
+		self._instance = None
+
+	def __call__(self):
+		if self._instance is None:
+			self._instance = self._factory(*self._args, **self._kwargs)
+		return self._instance
+
+	def __getattr__(self, name):
+		if self._instance is None:
+			self._instance = self._factory(*self._args, **self._kwargs)
+		return getattr(self._instance, name)
+
+
+embedding_models = {
+	"openai": LazyInit(
+		OpenAIEmbedding
+	),  # default model is OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002
+	"openai_embed_3_large": LazyInit(
+		OpenAIEmbedding, model_name=OpenAIEmbeddingModelType.TEXT_EMBED_3_LARGE
+	),
+	"openai_embed_3_small": LazyInit(
+		OpenAIEmbedding, model_name=OpenAIEmbeddingModelType.TEXT_EMBED_3_SMALL
+	),
+	# you can use your own model in this way.
+	"huggingface_baai_bge_small": LazyInit(
+		HuggingFaceEmbedding, model_name="BAAI/bge-small-en-v1.5"
+	),
+	"huggingface_cointegrated_rubert_tiny2": LazyInit(
+		HuggingFaceEmbedding, model_name="cointegrated/rubert-tiny2"
+	),
+	"huggingface_all_mpnet_base_v2": LazyInit(
+		HuggingFaceEmbedding,
+		model_name="sentence-transformers/all-mpnet-base-v2",
+		max_length=512,
+	),
+	"huggingface_bge_m3": LazyInit(HuggingFaceEmbedding, model_name="BAAI/bge-m3"),
+}
+
+generator_models = {
+	"openai": OpenAI,
+	"huggingfacellm": HuggingFaceLLM,
+	"openailike": OpenAILike,
+	"ollama": Ollama,
+	"mock": MockLLM,
+}
+
+rich_format = "[%(filename)s:%(lineno)s] >> %(message)s"
+logging.basicConfig(
+	level="INFO", format=rich_format, handlers=[RichHandler(rich_tracebacks=True)]
+)
+logger = logging.getLogger("AutoRAG")
+
+
+def handle_exception(exc_type, exc_value, exc_traceback):
+	logger = logging.getLogger("AutoRAG")
+	logger.error("Unexpected exception", exc_info=(exc_type, exc_value, exc_traceback))
+
+
+sys.excepthook = handle_exception
+
+transformers.logging.set_verbosity_error()

autorag-0.2.15/autorag/cli.py

@@ -0,0 +1,178 @@
+import importlib.resources
+import logging
+import os
+import pathlib
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+import click
+
+from autorag import dashboard
+from autorag.deploy import Runner
+from autorag.deploy import extract_best_config as original_extract_best_config
+from autorag.evaluator import Evaluator
+from autorag.validator import Validator
+
+logger = logging.getLogger("AutoRAG")
+
+
+@click.group()
+def cli():
+	pass
+
+
+@click.command()
+@click.option(
+	"--config",
+	"-c",
+	help="Path to config yaml file. Must be yaml or yml file.",
+	type=str,
+)
+@click.option(
+	"--qa_data_path", help="Path to QA dataset. Must be parquet file.", type=str
+)
+@click.option(
+	"--corpus_data_path", help="Path to corpus dataset. Must be parquet file.", type=str
+)
+@click.option(
+	"--project_dir", help="Path to project directory.", type=str, default=None
+)
+def evaluate(config, qa_data_path, corpus_data_path, project_dir):
+	if not config.endswith(".yaml") and not config.endswith(".yml"):
+		raise ValueError(f"Config file {config} is not a parquet file.")
+	if not os.path.exists(config):
+		raise ValueError(f"Config file {config} does not exist.")
+	evaluator = Evaluator(qa_data_path, corpus_data_path, project_dir=project_dir)
+	evaluator.start_trial(config)
+
+
+@click.command()
+@click.option("--config_path", type=str, help="Path to extracted config yaml file.")
+@click.option("--host", type=str, default="0.0.0.0", help="Host address")
+@click.option("--port", type=int, default=8000, help="Port number")
+@click.option(
+	"--project_dir", help="Path to project directory.", type=str, default=None
+)
+def run_api(config_path, host, port, project_dir):
+	runner = Runner.from_yaml(config_path, project_dir=project_dir)
+	logger.info(f"Running API server at {host}:{port}...")
+	runner.run_api_server(host, port)
+
+
+@click.command()
+@click.option(
+	"--yaml_path", type=click.Path(path_type=Path), help="Path to the YAML file."
+)
+@click.option(
+	"--project_dir",
+	type=click.Path(path_type=Path),
+	help="Path to the project directory.",
+)
+@click.option(
+	"--trial_path", type=click.Path(path_type=Path), help="Path to the trial directory."
+)
+def run_web(
+	yaml_path: Optional[str], project_dir: Optional[str], trial_path: Optional[str]
+):
+	try:
+		with importlib.resources.path("autorag", "web.py") as web_path:
+			web_py_path = str(web_path)
+	except ImportError:
+		raise ImportError(
+			"Could not locate the web.py file within the autorag package."
+			" Please ensure that autorag is correctly installed."
+		)
+
+	if not yaml_path and not trial_path:
+		raise ValueError("yaml_path or trial_path must be given.")
+	elif yaml_path and trial_path:
+		raise ValueError("yaml_path and trial_path cannot be given at the same time.")
+	elif yaml_path and not project_dir:
+		subprocess.run(
+			["streamlit", "run", web_py_path, "--", "--yaml_path", yaml_path]
+		)
+	elif yaml_path and project_dir:
+		subprocess.run(
+			[
+				"streamlit",
+				"run",
+				web_py_path,
+				"--",
+				"--yaml_path",
+				yaml_path,
+				"--project_dir",
+				project_dir,
+			]
+		)
+	elif trial_path:
+		subprocess.run(
+			["streamlit", "run", web_py_path, "--", "--trial_path", trial_path]
+		)
+
+
+@click.command()
+@click.option(
+	"--trial_dir",
+	type=click.Path(dir_okay=True, file_okay=False, exists=True),
+	required=True,
+)
+def run_dashboard(trial_dir: str):
+	dashboard.run(trial_dir)
+
+
+@click.command()
+@click.option("--trial_path", type=click.Path(), help="Path to the trial directory.")
+@click.option(
+	"--output_path",
+	type=click.Path(),
+	help="Path to the output directory." " Must be .yaml or .yml file.",
+)
+def extract_best_config(trial_path: str, output_path: str):
+	original_extract_best_config(trial_path, output_path)
+
+
+@click.command()
+@click.option("--trial_path", help="Path to trial directory.", type=str)
+def restart_evaluate(trial_path):
+	if not os.path.exists(trial_path):
+		raise ValueError(f"trial_path {trial_path} does not exist.")
+	project_dir = pathlib.PurePath(trial_path).parent
+	qa_data_path = os.path.join(project_dir, "data", "qa.parquet")
+	corpus_data_path = os.path.join(project_dir, "data", "corpus.parquet")
+	evaluator = Evaluator(qa_data_path, corpus_data_path, project_dir)
+	evaluator.restart_trial(trial_path)
+
+
+@click.command()
+@click.option(
+	"--config",
+	"-c",
+	help="Path to config yaml file. Must be yaml or yml file.",
+	type=str,
+)
+@click.option(
+	"--qa_data_path", help="Path to QA dataset. Must be parquet file.", type=str
+)
+@click.option(
+	"--corpus_data_path", help="Path to corpus dataset. Must be parquet file.", type=str
+)
+def validate(config, qa_data_path, corpus_data_path):
+	if not config.endswith(".yaml") and not config.endswith(".yml"):
+		raise ValueError(f"Config file {config} is not a parquet file.")
+	if not os.path.exists(config):
+		raise ValueError(f"Config file {config} does not exist.")
+	validator = Validator(qa_data_path=qa_data_path, corpus_data_path=corpus_data_path)
+	validator.validate(config)
+
+
+cli.add_command(evaluate, "evaluate")
+cli.add_command(run_api, "run_api")
+cli.add_command(run_web, "run_web")
+cli.add_command(run_dashboard, "dashboard")
+cli.add_command(extract_best_config, "extract_best_config")
+cli.add_command(restart_evaluate, "restart_evaluate")
+cli.add_command(validate, "validate")
+
+if __name__ == "__main__":
+	cli()