AutoRAG 0.2.14__tar.gz → 0.2.16__tar.gz

{autorag-0.2.14 → autorag-0.2.16}/.github/workflows/test.yml

@@ -26,12 +26,26 @@ jobs:
       - name: Install gcc
         run: |
           sudo apt-get install gcc
+      - name: Install poppler-utils
+        run: |
+          sudo apt-get install poppler-utils
+      - name: Install tesseract
+        run: |
+          sudo apt-get install tesseract-ocr
       - name: Install AutoRAG
         run: |
           pip install -e '.[all]'
       - name: Install dependencies
         run: |
           pip install -r tests/requirements.txt
+      - name: Upgrade pyOpenSSL
+        run: |
+          pip install --upgrade pyOpenSSL
+      - name: Install NLTK and download model
+        run: |
+          pip install nltk
+          python3 -c "import nltk; nltk.download('punkt_tab')"
+          python3 -c "import nltk; nltk.download('averaged_perceptron_tagger_eng')"
       - name: delete tests package
         run: python3 tests/delete_tests.py
       - name: Run tests

autorag-0.2.16/.pre-commit-config.yaml

@@ -0,0 +1,14 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0  # Use the ref you want to point at
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.6.1
+    hooks:
+      - id: ruff
+        types_or: [ python, pyi ]
+        args: [ "--fix" ]
+      - id: ruff-format
+        types_or: [ python, pyi ]

{autorag-0.2.14 → autorag-0.2.16}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.14
+Version: 0.2.16
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -236,8 +236,7 @@ Requires-Dist: rouge_score
 Requires-Dist: rich
 Requires-Dist: chromadb>=0.5.0
 Requires-Dist: click
-Requires-Dist: fastapi
-Requires-Dist: uvicorn
+Requires-Dist: Flask
 Requires-Dist: torch
 Requires-Dist: sentencepiece
 Requires-Dist: guidance
@@ -250,8 +249,8 @@ Requires-Dist: FlagEmbedding
 Requires-Dist: ragas
 Requires-Dist: llmlingua
 Requires-Dist: peft
-Requires-Dist: llama-index>=0.10.1
-Requires-Dist: llama-index-core>=0.10.1
+Requires-Dist: llama-index>=0.11.0
+Requires-Dist: llama-index-core>=0.11.0
 Requires-Dist: llama-index-readers-file
 Requires-Dist: llama-index-embeddings-openai
 Requires-Dist: llama-index-embeddings-huggingface
@@ -261,17 +260,34 @@ Requires-Dist: llama-index-llms-openai-like
 Requires-Dist: llama-index-llms-ollama
 Requires-Dist: llama-index-retrievers-bm25
 Requires-Dist: streamlit
+Requires-Dist: gradio
 Requires-Dist: langchain-core>=0.1.6
+Requires-Dist: langchain_unstructured
+Requires-Dist: langchain-upstage
 Requires-Dist: panel
 Requires-Dist: seaborn
 Requires-Dist: ipykernel
 Requires-Dist: ipywidgets
 Requires-Dist: ipywidgets_bokeh
 Provides-Extra: ko
-Requires-Dist: kiwipiepy; extra == "ko"
+Requires-Dist: kiwipiepy>=0.18.0; extra == "ko"
 Requires-Dist: konlpy; extra == "ko"
+Provides-Extra: dev
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Provides-Extra: parse
+Requires-Dist: PyMuPDF; extra == "parse"
+Requires-Dist: pdfminer.six; extra == "parse"
+Requires-Dist: pdfplumber; extra == "parse"
+Requires-Dist: unstructured; extra == "parse"
+Requires-Dist: jq; extra == "parse"
+Requires-Dist: unstructured[pdf]; extra == "parse"
+Requires-Dist: PyPDF2<3.0; extra == "parse"
+Requires-Dist: pdf2image; extra == "parse"
 Provides-Extra: all
 Requires-Dist: AutoRAG[ko]; extra == "all"
+Requires-Dist: AutoRAG[dev]; extra == "all"
+Requires-Dist: AutoRAG[parse]; extra == "all"
 
 # AutoRAG
 

{autorag-0.2.14 → autorag-0.2.16}/AutoRAG.egg-info/SOURCES.txt

@@ -1,4 +1,5 @@
 .gitignore
+.pre-commit-config.yaml
 CODE_OF_CONDUCT.md
 CONTRIBUTING.md
 LICENSE
@@ -20,19 +21,48 @@ AutoRAG.egg-info/requires.txt
 AutoRAG.egg-info/top_level.txt
 autorag/VERSION
 autorag/__init__.py
+autorag/chunker.py
 autorag/cli.py
 autorag/dashboard.py
 autorag/deploy.py
 autorag/evaluator.py
 autorag/node_line.py
+autorag/parser.py
 autorag/strategy.py
 autorag/support.py
 autorag/validator.py
 autorag/web.py
 autorag/data/__init__.py
+autorag/data/beta/__init__.py
+autorag/data/beta/sample.py
+autorag/data/beta/schema.py
+autorag/data/beta/filter/__init__.py
+autorag/data/beta/filter/dontknow.py
+autorag/data/beta/filter/prompt.py
+autorag/data/beta/generation_gt/__init__.py
+autorag/data/beta/generation_gt/base.py
+autorag/data/beta/generation_gt/llama_index_gen_gt.py
+autorag/data/beta/generation_gt/openai_gen_gt.py
+autorag/data/beta/generation_gt/prompt.py
+autorag/data/beta/query/__init__.py
+autorag/data/beta/query/llama_gen_query.py
+autorag/data/beta/query/openai_gen_query.py
+autorag/data/beta/query/prompt.py
+autorag/data/chunk/__init__.py
+autorag/data/chunk/base.py
+autorag/data/chunk/langchain_chunk.py
+autorag/data/chunk/llama_index_chunk.py
+autorag/data/chunk/run.py
 autorag/data/corpus/__init__.py
 autorag/data/corpus/langchain.py
 autorag/data/corpus/llama_index.py
+autorag/data/parse/__init__.py
+autorag/data/parse/base.py
+autorag/data/parse/clova.py
+autorag/data/parse/langchain_parse.py
+autorag/data/parse/llamaparse.py
+autorag/data/parse/run.py
+autorag/data/parse/table_hybrid_parse.py
 autorag/data/qacreation/__init__.py
 autorag/data/qacreation/base.py
 autorag/data/qacreation/llama_index.py
@@ -123,6 +153,7 @@ autorag/nodes/retrieval/hybrid_rrf.py
 autorag/nodes/retrieval/run.py
 autorag/nodes/retrieval/vectordb.py
 autorag/schema/__init__.py
+autorag/schema/metricinput.py
 autorag/schema/module.py
 autorag/schema/node.py
 autorag/utils/__init__.py
@@ -162,13 +193,19 @@ docs/source/_static/trial_folder.png
 docs/source/_static/trial_json.png
 docs/source/_static/trial_summary.png
 docs/source/_static/web_interface.png
+docs/source/_static/web_interface_gradio.png
 docs/source/_static/roadmap/RAG_paradigms.png
 docs/source/_static/roadmap/advanced_RAG.png
 docs/source/_static/roadmap/cycle.png
 docs/source/_static/roadmap/merger.png
 docs/source/_static/roadmap/node_line_modular.png
 docs/source/_static/roadmap/policy.png
+docs/source/api_spec/autorag.data.beta.generation_gt.rst
+docs/source/api_spec/autorag.data.beta.query.rst
+docs/source/api_spec/autorag.data.beta.rst
+docs/source/api_spec/autorag.data.beta.schema.rst
 docs/source/api_spec/autorag.data.corpus.rst
+docs/source/api_spec/autorag.data.parse.rst
 docs/source/api_spec/autorag.data.qacreation.rst
 docs/source/api_spec/autorag.data.rst
 docs/source/api_spec/autorag.data.utils.rst
@@ -191,6 +228,10 @@ docs/source/api_spec/modules.rst
 docs/source/data_creation/data_format.md
 docs/source/data_creation/ragas.md
 docs/source/data_creation/tutorial.md
+docs/source/data_creation/beta/data_creation.md
+docs/source/data_creation/beta/filter.md
+docs/source/data_creation/beta/query_gen.md
+docs/source/data_creation/beta/tutorial.md
 docs/source/deploy/api_endpoint.md
 docs/source/deploy/web.md
 docs/source/evaluate_metrics/generation.md
@@ -252,6 +293,9 @@ sample_config/full.yaml
 sample_config/simple_local.yaml
 sample_config/simple_ollama.yaml
 sample_config/simple_openai.yaml
+sample_config/chunk/chunk_full.yaml
+sample_config/parse/parse_full.yaml
+sample_config/parse/parse_hybird.yaml
 sample_dataset/README.md
 sample_dataset/eli5/load_eli5_dataset.py
 sample_dataset/hotpotqa/load_hotpotqa_dataset.py
@@ -261,17 +305,39 @@ tests/conftest.py
 tests/delete_tests.py
 tests/mock.py
 tests/requirements.txt
+tests/autorag/test_chunker.py
 tests/autorag/test_cli.py
 tests/autorag/test_dashboard.py
 tests/autorag/test_deploy.py
 tests/autorag/test_evaluator.py
+tests/autorag/test_parser.py
 tests/autorag/test_strategy.py
 tests/autorag/test_support.py
 tests/autorag/test_validator.py
 tests/autorag/test_web.py
+tests/autorag/data/beta/test_data_creation_piepline.py
+tests/autorag/data/beta/test_sample.py
+tests/autorag/data/beta/test_schema.py
+tests/autorag/data/beta/filter/test_dontknow.py
+tests/autorag/data/beta/generation_gt/base_test_generation_gt.py
+tests/autorag/data/beta/generation_gt/test_llama_index_gen_gt.py
+tests/autorag/data/beta/generation_gt/test_openai_gen_gt.py
+tests/autorag/data/beta/query/base_test_query_gen.py
+tests/autorag/data/beta/query/test_llama_gen_query.py
+tests/autorag/data/beta/query/test_openai_gen_query.py
+tests/autorag/data/chunk/test_chunk_base.py
+tests/autorag/data/chunk/test_chunk_run.py
+tests/autorag/data/chunk/test_langchain_chunk.py
+tests/autorag/data/chunk/test_llama_index_chunk.py
 tests/autorag/data/corpus/test_base.py
 tests/autorag/data/corpus/test_langchain.py
 tests/autorag/data/corpus/test_llama_index_corpus.py
+tests/autorag/data/parse/test_clova.py
+tests/autorag/data/parse/test_langchain_parse.py
+tests/autorag/data/parse/test_llamaparse.py
+tests/autorag/data/parse/test_parse_base.py
+tests/autorag/data/parse/test_parse_run.py
+tests/autorag/data/parse/test_table_hybrid_parse.py
 tests/autorag/data/qacreation/test_base_qacreation.py
 tests/autorag/data/qacreation/test_llama_index_qacreation.py
 tests/autorag/data/qacreation/test_ragas_qa_creation.py
@@ -339,6 +405,7 @@ tests/autorag/nodes/retrieval/test_hybrid_rrf.py
 tests/autorag/nodes/retrieval/test_retrieval_base.py
 tests/autorag/nodes/retrieval/test_run_retrieval_node.py
 tests/autorag/nodes/retrieval/test_vectordb.py
+tests/autorag/schema/test_metricinput_schema.py
 tests/autorag/schema/test_module_schema.py
 tests/autorag/schema/test_node_schema.py
 tests/autorag/utils/test_preprocess.py
@@ -350,10 +417,30 @@ tests/resources/qa_data_sample.parquet
 tests/resources/qa_test_data_sample.parquet
 tests/resources/sample_contents_nqa.csv
 tests/resources/simple.yaml
+tests/resources/simple_chunk.yaml
+tests/resources/simple_mock.yaml
+tests/resources/simple_parse.yaml
 tests/resources/test_bm25_retrieval.pkl
+tests/resources/chunk_data/sample_parsed.parquet
 tests/resources/data_creation/raw_dir/sample1.txt
 tests/resources/data_creation/raw_dir/sample2.txt
 tests/resources/data_creation/raw_dir/sample3.txt
+tests/resources/parse_data/all_files/baseball_1.pdf
+tests/resources/parse_data/all_files/csv_sample.csv
+tests/resources/parse_data/clova_data/result_sample.json
+tests/resources/parse_data/clova_data/result_table.txt
+tests/resources/parse_data/clova_data/result_text.txt
+tests/resources/parse_data/csv_data/csv_sample.csv
+tests/resources/parse_data/eng_text/baseball_1.pdf
+tests/resources/parse_data/eng_text/baseball_2.pdf
+tests/resources/parse_data/html_data/html_sample.html
+tests/resources/parse_data/hybrid_data/nfl_rulebook_both.pdf
+tests/resources/parse_data/json_data/json_sample.json
+tests/resources/parse_data/korean_table/only_table/kbo_only_table.pdf
+tests/resources/parse_data/korean_table/table_text/kbo_table_text.pdf
+tests/resources/parse_data/korean_text/korean_texts_two_page.pdf
+tests/resources/parse_data/markdown_data/markdown_sample.md
+tests/resources/parse_data/xml_data/xml_sample.xml
 tests/resources/qa_gen_prompts/prompt1.txt
 tests/resources/qa_gen_prompts/prompt2.txt
 tests/resources/qa_gen_prompts/prompt3.txt

{autorag-0.2.14 → autorag-0.2.16}/AutoRAG.egg-info/requires.txt

@@ -14,8 +14,7 @@ rouge_score
 rich
 chromadb>=0.5.0
 click
-fastapi
-uvicorn
+Flask
 torch
 sentencepiece
 guidance
@@ -28,8 +27,8 @@ FlagEmbedding
 ragas
 llmlingua
 peft
-llama-index>=0.10.1
-llama-index-core>=0.10.1
+llama-index>=0.11.0
+llama-index-core>=0.11.0
 llama-index-readers-file
 llama-index-embeddings-openai
 llama-index-embeddings-huggingface
@@ -39,7 +38,10 @@ llama-index-llms-openai-like
 llama-index-llms-ollama
 llama-index-retrievers-bm25
 streamlit
+gradio
 langchain-core>=0.1.6
+langchain_unstructured
+langchain-upstage
 panel
 seaborn
 ipykernel
@@ -48,7 +50,23 @@ ipywidgets_bokeh
 
 [all]
 AutoRAG[ko]
+AutoRAG[dev]
+AutoRAG[parse]
+
+[dev]
+ruff
+pre-commit
 
 [ko]
-kiwipiepy
+kiwipiepy>=0.18.0
 konlpy
+
+[parse]
+PyMuPDF
+pdfminer.six
+pdfplumber
+unstructured
+jq
+unstructured[pdf]
+PyPDF2<3.0
+pdf2image

autorag-0.2.16/CONTRIBUTING.md

@@ -0,0 +1,106 @@
+# Getting Started
+
+Thank you so much for your consideration of contributing to AutoRAG!
+Okay Let's get started!
+
+## Quickstart
+
+1. Fork the repo and clone your fork
+2. Create a new virtual environment and activate it
+3. Install the dependencies and package (`pip install -e '.[all]'`)
+4. Install docs and test dependencies (`pip install -r docs/requirements.txt` and `pip install -r tests/requirements.txt`)
+
+Also, join our Discord at [here](https://discord.gg/P4DYXfmSAs) and discuss about your contribution with the community.
+
+## Contribution Guidelines
+
+### What should I work on?
+
+1. Add new modules or nodes to the AutoRAG.
+2. Fix bugs
+3. Add new feature to the AutoRAG
+4. Add new sample config YAML files
+5. Add new sample dataset
+6. Improve code quality & documentation
+7. Report a bug or suggest new feature
+
+It can be great you search 'good first issue' tag on github issue tab.
+
+# Development Guidelines
+
+## Repo Structure
+
+The `AutoRAG` repo have a standard python library structure.
+
+You can find the core part of the library in `autorag` folder.
+You can find the test codes of the `AutoRAG` at the `tests` folder.
+Plus, you can find the [AutoRAG docs](https://docs.auto-rag.com) at the `docs` folder.
+
+For resources, you can find sample config YAML files at `sample_config` folder.
+Plus, you can find sample dataset download script from `sample_dataset` folder.
+
+## Setting up environment
+
+As a python package, AutoRAG tested primarily in Python versions >= 3.9.
+Here's a guide to set environment.
+
+1. Fork [AutoRAG Github repo](https://github.com/Marker-Inc-Korea/AutoRAG) and clone it to your local machine. (New to GitHub / git? Here's [how](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo).)
+2. Open your cloned repository to your IDE or open your project at the terminal through `cd`.
+3. Install pre-commit hooks by running `pre-commit install` in the terminal.
+4. Make a new virtual environment (Highly Recommended)
+5. Install AutoRAG as development version. `pip install -e '.[all]'`
+6. Install test dependencies `pip install -r tests/requirements.txt`
+7. Write `pytest.ini` file and add env variable for running tests. (We do not need OPENAI_API_KEY at GitHub actions)
+```ini
+[pytest]
+env =
+    OPENAI_API_KEY=sk-xxxx
+
+log_cli=true
+log_cli_level=INFO
+```
+8. Install docs dependencies `pip install -r docs/requirements.txt`
+
+## Validating your change
+
+Let's make sure to `format/lint` our change.
+For bigger changes, let's also make sure to `test` our change.
+
+### Formatting/Linting
+
+We use `ruff` to linting and formatting our code.
+If you have installed pre-commit hooks in this repo, they should have taken care of the formatting and linting automatically.
+(Use `pre-commit install`)
+
+If you want to do it manually, you can lint and format with ruff.
+
+```bash
+ruff check --fix
+ruff format
+```
+
+### Testing
+
+If you modified or added new code logic, you have to create tests for them, too.
+- In almost all cases, add unit tests.
+- If your new features uses heavy GPU models or outside API connections, you must use mock and pytest fixtures.
+- The added test case must not leave any extra files.
+- You must cover all cases of features or adjust that you made.
+- If you don't know well about `pytest`, `pytest-env` and pytest fixtures and mock, please learn about it.
+- Lastly, if the test you made must need CUDA-enabled device, you can mark the test with `@pytest.mark.skipif(is_github_action())` so we can skip it on github actions.
+
+Plus, your test will run as github actions when you try to merge it to the main branch.
+
+#### Warning about OPENAI_API_KEY
+
+When executing test at github actions, it will be failed due to the OpenAI API key.
+Don't worry, we will run the test locally and merge your code if there is no problem.
+
+## Making a Pull Request
+
+You can make a pull request to the main branch of the AutoRAG repo.
+At the fork repository, you will make a PR to AutoRAG repo.
+You can assign the issue with `close #issue_number` to automatically close the issue when the PR is merged.
+
+After that, please assign `vkehfdl1` or `bwook00` as a reviewer.
+For merge, you will have to get at least one approval from the reviewer.

{autorag-0.2.14 → autorag-0.2.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.14
+Version: 0.2.16
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -236,8 +236,7 @@ Requires-Dist: rouge_score
 Requires-Dist: rich
 Requires-Dist: chromadb>=0.5.0
 Requires-Dist: click
-Requires-Dist: fastapi
-Requires-Dist: uvicorn
+Requires-Dist: Flask
 Requires-Dist: torch
 Requires-Dist: sentencepiece
 Requires-Dist: guidance
@@ -250,8 +249,8 @@ Requires-Dist: FlagEmbedding
 Requires-Dist: ragas
 Requires-Dist: llmlingua
 Requires-Dist: peft
-Requires-Dist: llama-index>=0.10.1
-Requires-Dist: llama-index-core>=0.10.1
+Requires-Dist: llama-index>=0.11.0
+Requires-Dist: llama-index-core>=0.11.0
 Requires-Dist: llama-index-readers-file
 Requires-Dist: llama-index-embeddings-openai
 Requires-Dist: llama-index-embeddings-huggingface
@@ -261,17 +260,34 @@ Requires-Dist: llama-index-llms-openai-like
 Requires-Dist: llama-index-llms-ollama
 Requires-Dist: llama-index-retrievers-bm25
 Requires-Dist: streamlit
+Requires-Dist: gradio
 Requires-Dist: langchain-core>=0.1.6
+Requires-Dist: langchain_unstructured
+Requires-Dist: langchain-upstage
 Requires-Dist: panel
 Requires-Dist: seaborn
 Requires-Dist: ipykernel
 Requires-Dist: ipywidgets
 Requires-Dist: ipywidgets_bokeh
 Provides-Extra: ko
-Requires-Dist: kiwipiepy; extra == "ko"
+Requires-Dist: kiwipiepy>=0.18.0; extra == "ko"
 Requires-Dist: konlpy; extra == "ko"
+Provides-Extra: dev
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: pre-commit; extra == "dev"
+Provides-Extra: parse
+Requires-Dist: PyMuPDF; extra == "parse"
+Requires-Dist: pdfminer.six; extra == "parse"
+Requires-Dist: pdfplumber; extra == "parse"
+Requires-Dist: unstructured; extra == "parse"
+Requires-Dist: jq; extra == "parse"
+Requires-Dist: unstructured[pdf]; extra == "parse"
+Requires-Dist: PyPDF2<3.0; extra == "parse"
+Requires-Dist: pdf2image; extra == "parse"
 Provides-Extra: all
 Requires-Dist: AutoRAG[ko]; extra == "all"
+Requires-Dist: AutoRAG[dev]; extra == "all"
+Requires-Dist: AutoRAG[parse]; extra == "all"
 
 # AutoRAG
 

autorag-0.2.16/autorag/VERSION

@@ -0,0 +1 @@
+0.2.16

autorag-0.2.16/autorag/__init__.py

@@ -0,0 +1,103 @@
+import logging
+import logging.config
+import os
+import sys
+from random import random
+from typing import List
+
+import transformers
+from llama_index.core import MockEmbedding
+from llama_index.core.llms.mock import MockLLM
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.embeddings.openai import OpenAIEmbedding
+from llama_index.embeddings.openai import OpenAIEmbeddingModelType
+from llama_index.llms.huggingface import HuggingFaceLLM
+from llama_index.llms.ollama import Ollama
+from llama_index.llms.openai import OpenAI
+from llama_index.llms.openai_like import OpenAILike
+from langchain.embeddings import OpenAIEmbeddings
+from rich.logging import RichHandler
+
+version_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "VERSION")
+
+with open(version_path, "r") as f:
+	__version__ = f.read().strip()
+
+
+class LazyInit:
+	def __init__(self, factory, *args, **kwargs):
+		self._factory = factory
+		self._args = args
+		self._kwargs = kwargs
+		self._instance = None
+
+	def __call__(self):
+		if self._instance is None:
+			self._instance = self._factory(*self._args, **self._kwargs)
+		return self._instance
+
+	def __getattr__(self, name):
+		if self._instance is None:
+			self._instance = self._factory(*self._args, **self._kwargs)
+		return getattr(self._instance, name)
+
+
+class MockEmbeddingRandom(MockEmbedding):
+	"""Mock embedding with random vectors."""
+
+	def _get_vector(self) -> List[float]:
+		return [random() for _ in range(self.embed_dim)]
+
+
+embedding_models = {
+	# llama index
+	"openai": LazyInit(
+		OpenAIEmbedding
+	),  # default model is OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002
+	"openai_embed_3_large": LazyInit(
+		OpenAIEmbedding, model_name=OpenAIEmbeddingModelType.TEXT_EMBED_3_LARGE
+	),
+	"openai_embed_3_small": LazyInit(
+		OpenAIEmbedding, model_name=OpenAIEmbeddingModelType.TEXT_EMBED_3_SMALL
+	),
+	# you can use your own model in this way.
+	"huggingface_baai_bge_small": LazyInit(
+		HuggingFaceEmbedding, model_name="BAAI/bge-small-en-v1.5"
+	),
+	"huggingface_cointegrated_rubert_tiny2": LazyInit(
+		HuggingFaceEmbedding, model_name="cointegrated/rubert-tiny2"
+	),
+	"huggingface_all_mpnet_base_v2": LazyInit(
+		HuggingFaceEmbedding,
+		model_name="sentence-transformers/all-mpnet-base-v2",
+		max_length=512,
+	),
+	"huggingface_bge_m3": LazyInit(HuggingFaceEmbedding, model_name="BAAI/bge-m3"),
+	"mock": LazyInit(MockEmbeddingRandom, embed_dim=768),
+	# langchain
+	"openai_langchain": LazyInit(OpenAIEmbeddings),
+}
+
+generator_models = {
+	"openai": OpenAI,
+	"huggingfacellm": HuggingFaceLLM,
+	"openailike": OpenAILike,
+	"ollama": Ollama,
+	"mock": MockLLM,
+}
+
+rich_format = "[%(filename)s:%(lineno)s] >> %(message)s"
+logging.basicConfig(
+	level="INFO", format=rich_format, handlers=[RichHandler(rich_tracebacks=True)]
+)
+logger = logging.getLogger("AutoRAG")
+
+
+def handle_exception(exc_type, exc_value, exc_traceback):
+	logger = logging.getLogger("AutoRAG")
+	logger.error("Unexpected exception", exc_info=(exc_type, exc_value, exc_traceback))
+
+
+sys.excepthook = handle_exception
+
+transformers.logging.set_verbosity_error()