AutoRAG 0.2.15__tar.gz → 0.2.17__tar.gz

{autorag-0.2.15 → autorag-0.2.17}/.github/workflows/test.yml

@@ -26,12 +26,26 @@ jobs:
       - name: Install gcc
         run: |
           sudo apt-get install gcc
+      - name: Install poppler-utils
+        run: |
+          sudo apt-get install poppler-utils
+      - name: Install tesseract
+        run: |
+          sudo apt-get install tesseract-ocr
       - name: Install AutoRAG
         run: |
           pip install -e '.[all]'
       - name: Install dependencies
         run: |
           pip install -r tests/requirements.txt
+      - name: Upgrade pyOpenSSL
+        run: |
+          pip install --upgrade pyOpenSSL
+      - name: Install NLTK and download model
+        run: |
+          pip install nltk
+          python3 -c "import nltk; nltk.download('punkt_tab')"
+          python3 -c "import nltk; nltk.download('averaged_perceptron_tagger_eng')"
       - name: delete tests package
         run: python3 tests/delete_tests.py
       - name: Run tests

{autorag-0.2.15 → autorag-0.2.17}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.15
+Version: 0.2.17
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -236,8 +236,7 @@ Requires-Dist: rouge_score
 Requires-Dist: rich
 Requires-Dist: chromadb>=0.5.0
 Requires-Dist: click
-Requires-Dist: fastapi
-Requires-Dist: uvicorn
+Requires-Dist: Flask
 Requires-Dist: torch
 Requires-Dist: sentencepiece
 Requires-Dist: guidance
@@ -263,20 +262,32 @@ Requires-Dist: llama-index-retrievers-bm25
 Requires-Dist: streamlit
 Requires-Dist: gradio
 Requires-Dist: langchain-core>=0.1.6
+Requires-Dist: langchain_unstructured
+Requires-Dist: langchain-upstage
 Requires-Dist: panel
 Requires-Dist: seaborn
 Requires-Dist: ipykernel
 Requires-Dist: ipywidgets
 Requires-Dist: ipywidgets_bokeh
 Provides-Extra: ko
-Requires-Dist: kiwipiepy; extra == "ko"
+Requires-Dist: kiwipiepy>=0.18.0; extra == "ko"
 Requires-Dist: konlpy; extra == "ko"
 Provides-Extra: dev
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
+Provides-Extra: parse
+Requires-Dist: PyMuPDF; extra == "parse"
+Requires-Dist: pdfminer.six; extra == "parse"
+Requires-Dist: pdfplumber; extra == "parse"
+Requires-Dist: unstructured; extra == "parse"
+Requires-Dist: jq; extra == "parse"
+Requires-Dist: unstructured[pdf]; extra == "parse"
+Requires-Dist: PyPDF2<3.0; extra == "parse"
+Requires-Dist: pdf2image; extra == "parse"
 Provides-Extra: all
 Requires-Dist: AutoRAG[ko]; extra == "all"
 Requires-Dist: AutoRAG[dev]; extra == "all"
+Requires-Dist: AutoRAG[parse]; extra == "all"
 
 # AutoRAG
 

{autorag-0.2.15 → autorag-0.2.17}/AutoRAG.egg-info/SOURCES.txt

@@ -21,19 +21,49 @@ AutoRAG.egg-info/requires.txt
 AutoRAG.egg-info/top_level.txt
 autorag/VERSION
 autorag/__init__.py
+autorag/chunker.py
 autorag/cli.py
 autorag/dashboard.py
 autorag/deploy.py
 autorag/evaluator.py
 autorag/node_line.py
+autorag/parser.py
 autorag/strategy.py
 autorag/support.py
 autorag/validator.py
 autorag/web.py
 autorag/data/__init__.py
+autorag/data/beta/__init__.py
+autorag/data/beta/extract_evidence.py
+autorag/data/beta/sample.py
+autorag/data/beta/schema.py
+autorag/data/beta/filter/__init__.py
+autorag/data/beta/filter/dontknow.py
+autorag/data/beta/filter/prompt.py
+autorag/data/beta/generation_gt/__init__.py
+autorag/data/beta/generation_gt/base.py
+autorag/data/beta/generation_gt/llama_index_gen_gt.py
+autorag/data/beta/generation_gt/openai_gen_gt.py
+autorag/data/beta/generation_gt/prompt.py
+autorag/data/beta/query/__init__.py
+autorag/data/beta/query/llama_gen_query.py
+autorag/data/beta/query/openai_gen_query.py
+autorag/data/beta/query/prompt.py
+autorag/data/chunk/__init__.py
+autorag/data/chunk/base.py
+autorag/data/chunk/langchain_chunk.py
+autorag/data/chunk/llama_index_chunk.py
+autorag/data/chunk/run.py
 autorag/data/corpus/__init__.py
 autorag/data/corpus/langchain.py
 autorag/data/corpus/llama_index.py
+autorag/data/parse/__init__.py
+autorag/data/parse/base.py
+autorag/data/parse/clova.py
+autorag/data/parse/langchain_parse.py
+autorag/data/parse/llamaparse.py
+autorag/data/parse/run.py
+autorag/data/parse/table_hybrid_parse.py
 autorag/data/qacreation/__init__.py
 autorag/data/qacreation/base.py
 autorag/data/qacreation/llama_index.py
@@ -124,6 +154,7 @@ autorag/nodes/retrieval/hybrid_rrf.py
 autorag/nodes/retrieval/run.py
 autorag/nodes/retrieval/vectordb.py
 autorag/schema/__init__.py
+autorag/schema/metricinput.py
 autorag/schema/module.py
 autorag/schema/node.py
 autorag/utils/__init__.py
@@ -170,7 +201,12 @@ docs/source/_static/roadmap/cycle.png
 docs/source/_static/roadmap/merger.png
 docs/source/_static/roadmap/node_line_modular.png
 docs/source/_static/roadmap/policy.png
+docs/source/api_spec/autorag.data.beta.generation_gt.rst
+docs/source/api_spec/autorag.data.beta.query.rst
+docs/source/api_spec/autorag.data.beta.rst
+docs/source/api_spec/autorag.data.beta.schema.rst
 docs/source/api_spec/autorag.data.corpus.rst
+docs/source/api_spec/autorag.data.parse.rst
 docs/source/api_spec/autorag.data.qacreation.rst
 docs/source/api_spec/autorag.data.rst
 docs/source/api_spec/autorag.data.utils.rst
@@ -193,6 +229,20 @@ docs/source/api_spec/modules.rst
 docs/source/data_creation/data_format.md
 docs/source/data_creation/ragas.md
 docs/source/data_creation/tutorial.md
+docs/source/data_creation/beta/data_creation.md
+docs/source/data_creation/beta/tutorial.md
+docs/source/data_creation/beta/chunk/chunk.md
+docs/source/data_creation/beta/chunk/langchain_chunk.md
+docs/source/data_creation/beta/chunk/llama_index_chunk.md
+docs/source/data_creation/beta/parse/clova.md
+docs/source/data_creation/beta/parse/langchain_parse.md
+docs/source/data_creation/beta/parse/llama_parse.md
+docs/source/data_creation/beta/parse/parse.md
+docs/source/data_creation/beta/parse/table_hybrid_parse.md
+docs/source/data_creation/beta/qa_creation/answer_gen.md
+docs/source/data_creation/beta/qa_creation/filter.md
+docs/source/data_creation/beta/qa_creation/qa_creation.md
+docs/source/data_creation/beta/qa_creation/query_gen.md
 docs/source/deploy/api_endpoint.md
 docs/source/deploy/web.md
 docs/source/evaluate_metrics/generation.md
@@ -254,6 +304,9 @@ sample_config/full.yaml
 sample_config/simple_local.yaml
 sample_config/simple_ollama.yaml
 sample_config/simple_openai.yaml
+sample_config/chunk/chunk_full.yaml
+sample_config/parse/parse_full.yaml
+sample_config/parse/parse_hybird.yaml
 sample_dataset/README.md
 sample_dataset/eli5/load_eli5_dataset.py
 sample_dataset/hotpotqa/load_hotpotqa_dataset.py
@@ -263,17 +316,39 @@ tests/conftest.py
 tests/delete_tests.py
 tests/mock.py
 tests/requirements.txt
+tests/autorag/test_chunker.py
 tests/autorag/test_cli.py
 tests/autorag/test_dashboard.py
 tests/autorag/test_deploy.py
 tests/autorag/test_evaluator.py
+tests/autorag/test_parser.py
 tests/autorag/test_strategy.py
 tests/autorag/test_support.py
 tests/autorag/test_validator.py
 tests/autorag/test_web.py
+tests/autorag/data/beta/test_data_creation_piepline.py
+tests/autorag/data/beta/test_sample.py
+tests/autorag/data/beta/test_schema.py
+tests/autorag/data/beta/filter/test_dontknow.py
+tests/autorag/data/beta/generation_gt/base_test_generation_gt.py
+tests/autorag/data/beta/generation_gt/test_llama_index_gen_gt.py
+tests/autorag/data/beta/generation_gt/test_openai_gen_gt.py
+tests/autorag/data/beta/query/base_test_query_gen.py
+tests/autorag/data/beta/query/test_llama_gen_query.py
+tests/autorag/data/beta/query/test_openai_gen_query.py
+tests/autorag/data/chunk/test_chunk_base.py
+tests/autorag/data/chunk/test_chunk_run.py
+tests/autorag/data/chunk/test_langchain_chunk.py
+tests/autorag/data/chunk/test_llama_index_chunk.py
 tests/autorag/data/corpus/test_base.py
 tests/autorag/data/corpus/test_langchain.py
 tests/autorag/data/corpus/test_llama_index_corpus.py
+tests/autorag/data/parse/test_clova.py
+tests/autorag/data/parse/test_langchain_parse.py
+tests/autorag/data/parse/test_llamaparse.py
+tests/autorag/data/parse/test_parse_base.py
+tests/autorag/data/parse/test_parse_run.py
+tests/autorag/data/parse/test_table_hybrid_parse.py
 tests/autorag/data/qacreation/test_base_qacreation.py
 tests/autorag/data/qacreation/test_llama_index_qacreation.py
 tests/autorag/data/qacreation/test_ragas_qa_creation.py
@@ -341,6 +416,7 @@ tests/autorag/nodes/retrieval/test_hybrid_rrf.py
 tests/autorag/nodes/retrieval/test_retrieval_base.py
 tests/autorag/nodes/retrieval/test_run_retrieval_node.py
 tests/autorag/nodes/retrieval/test_vectordb.py
+tests/autorag/schema/test_metricinput_schema.py
 tests/autorag/schema/test_module_schema.py
 tests/autorag/schema/test_node_schema.py
 tests/autorag/utils/test_preprocess.py
@@ -352,10 +428,30 @@ tests/resources/qa_data_sample.parquet
 tests/resources/qa_test_data_sample.parquet
 tests/resources/sample_contents_nqa.csv
 tests/resources/simple.yaml
+tests/resources/simple_chunk.yaml
+tests/resources/simple_mock.yaml
+tests/resources/simple_parse.yaml
 tests/resources/test_bm25_retrieval.pkl
+tests/resources/chunk_data/sample_parsed.parquet
 tests/resources/data_creation/raw_dir/sample1.txt
 tests/resources/data_creation/raw_dir/sample2.txt
 tests/resources/data_creation/raw_dir/sample3.txt
+tests/resources/parse_data/all_files/baseball_1.pdf
+tests/resources/parse_data/all_files/csv_sample.csv
+tests/resources/parse_data/clova_data/result_sample.json
+tests/resources/parse_data/clova_data/result_table.txt
+tests/resources/parse_data/clova_data/result_text.txt
+tests/resources/parse_data/csv_data/csv_sample.csv
+tests/resources/parse_data/eng_text/baseball_1.pdf
+tests/resources/parse_data/eng_text/baseball_2.pdf
+tests/resources/parse_data/html_data/html_sample.html
+tests/resources/parse_data/hybrid_data/nfl_rulebook_both.pdf
+tests/resources/parse_data/json_data/json_sample.json
+tests/resources/parse_data/korean_table/only_table/kbo_only_table.pdf
+tests/resources/parse_data/korean_table/table_text/kbo_table_text.pdf
+tests/resources/parse_data/korean_text/korean_texts_two_page.pdf
+tests/resources/parse_data/markdown_data/markdown_sample.md
+tests/resources/parse_data/xml_data/xml_sample.xml
 tests/resources/qa_gen_prompts/prompt1.txt
 tests/resources/qa_gen_prompts/prompt2.txt
 tests/resources/qa_gen_prompts/prompt3.txt

{autorag-0.2.15 → autorag-0.2.17}/AutoRAG.egg-info/requires.txt

@@ -14,8 +14,7 @@ rouge_score
 rich
 chromadb>=0.5.0
 click
-fastapi
-uvicorn
+Flask
 torch
 sentencepiece
 guidance
@@ -41,6 +40,8 @@ llama-index-retrievers-bm25
 streamlit
 gradio
 langchain-core>=0.1.6
+langchain_unstructured
+langchain-upstage
 panel
 seaborn
 ipykernel
@@ -50,11 +51,22 @@ ipywidgets_bokeh
 [all]
 AutoRAG[ko]
 AutoRAG[dev]
+AutoRAG[parse]
 
 [dev]
 ruff
 pre-commit
 
 [ko]
-kiwipiepy
+kiwipiepy>=0.18.0
 konlpy
+
+[parse]
+PyMuPDF
+pdfminer.six
+pdfplumber
+unstructured
+jq
+unstructured[pdf]
+PyPDF2<3.0
+pdf2image

{autorag-0.2.15 → autorag-0.2.17}/CONTRIBUTING.md

@@ -50,7 +50,7 @@ Here's a guide to set environment.
 4. Make a new virtual environment (Highly Recommended)
 5. Install AutoRAG as development version. `pip install -e '.[all]'`
 6. Install test dependencies `pip install -r tests/requirements.txt`
-7. Write `pytest.ini` file and add env variable for running tests.
+7. Write `pytest.ini` file and add env variable for running tests. (We do not need OPENAI_API_KEY at GitHub actions)
 ```ini
 [pytest]
 env =

{autorag-0.2.15 → autorag-0.2.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.15
+Version: 0.2.17
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -236,8 +236,7 @@ Requires-Dist: rouge_score
 Requires-Dist: rich
 Requires-Dist: chromadb>=0.5.0
 Requires-Dist: click
-Requires-Dist: fastapi
-Requires-Dist: uvicorn
+Requires-Dist: Flask
 Requires-Dist: torch
 Requires-Dist: sentencepiece
 Requires-Dist: guidance
@@ -263,20 +262,32 @@ Requires-Dist: llama-index-retrievers-bm25
 Requires-Dist: streamlit
 Requires-Dist: gradio
 Requires-Dist: langchain-core>=0.1.6
+Requires-Dist: langchain_unstructured
+Requires-Dist: langchain-upstage
 Requires-Dist: panel
 Requires-Dist: seaborn
 Requires-Dist: ipykernel
 Requires-Dist: ipywidgets
 Requires-Dist: ipywidgets_bokeh
 Provides-Extra: ko
-Requires-Dist: kiwipiepy; extra == "ko"
+Requires-Dist: kiwipiepy>=0.18.0; extra == "ko"
 Requires-Dist: konlpy; extra == "ko"
 Provides-Extra: dev
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
+Provides-Extra: parse
+Requires-Dist: PyMuPDF; extra == "parse"
+Requires-Dist: pdfminer.six; extra == "parse"
+Requires-Dist: pdfplumber; extra == "parse"
+Requires-Dist: unstructured; extra == "parse"
+Requires-Dist: jq; extra == "parse"
+Requires-Dist: unstructured[pdf]; extra == "parse"
+Requires-Dist: PyPDF2<3.0; extra == "parse"
+Requires-Dist: pdf2image; extra == "parse"
 Provides-Extra: all
 Requires-Dist: AutoRAG[ko]; extra == "all"
 Requires-Dist: AutoRAG[dev]; extra == "all"
+Requires-Dist: AutoRAG[parse]; extra == "all"
 
 # AutoRAG
 

autorag-0.2.17/autorag/VERSION

@@ -0,0 +1 @@
+0.2.17

{autorag-0.2.15 → autorag-0.2.17}/autorag/__init__.py

@@ -2,8 +2,11 @@ import logging
 import logging.config
 import os
 import sys
+from random import random
+from typing import List
 
 import transformers
+from llama_index.core import MockEmbedding
 from llama_index.core.llms.mock import MockLLM
 from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 from llama_index.embeddings.openai import OpenAIEmbedding
@@ -12,6 +15,7 @@ from llama_index.llms.huggingface import HuggingFaceLLM
 from llama_index.llms.ollama import Ollama
 from llama_index.llms.openai import OpenAI
 from llama_index.llms.openai_like import OpenAILike
+from langchain.embeddings import OpenAIEmbeddings
 from rich.logging import RichHandler
 
 version_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "VERSION")
@@ -38,7 +42,15 @@ class LazyInit:
 		return getattr(self._instance, name)
 
 
+class MockEmbeddingRandom(MockEmbedding):
+	"""Mock embedding with random vectors."""
+
+	def _get_vector(self) -> List[float]:
+		return [random() for _ in range(self.embed_dim)]
+
+
 embedding_models = {
+	# llama index
 	"openai": LazyInit(
 		OpenAIEmbedding
 	),  # default model is OpenAIEmbeddingModelType.TEXT_EMBED_ADA_002
@@ -61,6 +73,9 @@ embedding_models = {
 		max_length=512,
 	),
 	"huggingface_bge_m3": LazyInit(HuggingFaceEmbedding, model_name="BAAI/bge-m3"),
+	"mock": LazyInit(MockEmbeddingRandom, embed_dim=768),
+	# langchain
+	"openai_langchain": LazyInit(OpenAIEmbeddings),
 }
 
 generator_models = {

autorag-0.2.17/autorag/chunker.py

@@ -0,0 +1,81 @@
+import json
+import logging
+import os
+import shutil
+from datetime import datetime
+from typing import Optional
+
+import pandas as pd
+
+from autorag.data.chunk.run import run_chunker
+from autorag.data.utils.util import load_yaml, get_param_combinations
+
+logger = logging.getLogger("AutoRAG")
+
+
+class Chunker:
+	def __init__(self, raw_df: pd.DataFrame, project_dir: Optional[str] = None):
+		self.parsed_raw = raw_df
+		self.project_dir = project_dir if project_dir is not None else os.getcwd()
+
+	@classmethod
+	def from_parquet(
+		cls, parsed_data_path: str, project_dir: Optional[str] = None
+	) -> "Chunker":
+		if not os.path.exists(parsed_data_path):
+			raise ValueError(f"parsed_data_path {parsed_data_path} does not exist.")
+		if not parsed_data_path.endswith("parquet"):
+			raise ValueError(
+				f"parsed_data_path {parsed_data_path} is not a parquet file."
+			)
+		parsed_result = pd.read_parquet(parsed_data_path, engine="pyarrow")
+		return cls(parsed_result, project_dir)
+
+	def start_chunking(self, yaml_path: str):
+		trial_name = self.__get_new_trial_name()
+		self.__make_trial_dir(trial_name)
+
+		# Copy YAML file to the trial directory
+		shutil.copy(
+			yaml_path, os.path.join(self.project_dir, trial_name, "chunk_config.yaml")
+		)
+
+		# load yaml file
+		modules = load_yaml(yaml_path)
+
+		input_modules, input_params = get_param_combinations(modules)
+
+		logger.info("Chunking Start...")
+		run_chunker(
+			modules=input_modules,
+			module_params=input_params,
+			parsed_result=self.parsed_raw,
+			trial_path=os.path.join(self.project_dir, trial_name),
+		)
+		logger.info("Chunking Done!")
+
+	def __get_new_trial_name(self) -> str:
+		trial_json_path = os.path.join(self.project_dir, "trial.json")
+		if not os.path.exists(trial_json_path):
+			return "0"
+		with open(trial_json_path, "r") as f:
+			trial_json = json.load(f)
+		return str(int(trial_json[-1]["trial_name"]) + 1)
+
+	def __make_trial_dir(self, trial_name: str):
+		trial_json_path = os.path.join(self.project_dir, "trial.json")
+		if os.path.exists(trial_json_path):
+			with open(trial_json_path, "r") as f:
+				trial_json = json.load(f)
+		else:
+			trial_json = []
+
+		trial_json.append(
+			{
+				"trial_name": trial_name,
+				"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+			}
+		)
+		os.makedirs(os.path.join(self.project_dir, trial_name))
+		with open(trial_json_path, "w") as f:
+			json.dump(trial_json, f, indent=4)

{autorag-0.2.15 → autorag-0.2.17}/autorag/dashboard.py

@@ -167,7 +167,7 @@ def yaml_to_markdown(yaml_filepath):
 	with open(yaml_filepath, "r", encoding="utf-8") as file:
 		try:
 			content = yaml.safe_load(file)
-			markdown_content += f"## {os.path.basename(yaml_filepath)}\n```yaml\n{yaml.dump(content, allow_unicode=True)}\n```\n\n"
+			markdown_content += f"## {os.path.basename(yaml_filepath)}\n```yaml\n{yaml.safe_dump(content, allow_unicode=True)}\n```\n\n"
 		except yaml.YAMLError as exc:
 			print(f"Error in {yaml_filepath}: {exc}")
 	return markdown_content

autorag-0.2.17/autorag/data/__init__.py

@@ -0,0 +1,100 @@
+from typing import List, Callable
+
+from langchain_community.document_loaders import (
+	PDFMinerLoader,
+	PDFPlumberLoader,
+	PyPDFium2Loader,
+	PyPDFLoader,
+	PyMuPDFLoader,
+	UnstructuredPDFLoader,
+	CSVLoader,
+	JSONLoader,
+	UnstructuredMarkdownLoader,
+	BSHTMLLoader,
+	UnstructuredXMLLoader,
+	DirectoryLoader,
+)
+from langchain_unstructured import UnstructuredLoader
+from langchain_upstage import UpstageLayoutAnalysisLoader
+
+from llama_index.core.node_parser import (
+	TokenTextSplitter,
+	SentenceSplitter,
+	SentenceWindowNodeParser,
+	SemanticSplitterNodeParser,
+	SemanticDoubleMergingSplitterNodeParser,
+	SimpleFileNodeParser,
+)
+from langchain.text_splitter import (
+	RecursiveCharacterTextSplitter,
+	CharacterTextSplitter,
+	KonlpyTextSplitter,
+	SentenceTransformersTokenTextSplitter,
+)
+
+from autorag import LazyInit
+
+parse_modules = {
+	# PDF
+	"pdfminer": PDFMinerLoader,
+	"pdfplumber": PDFPlumberLoader,
+	"pypdfium2": PyPDFium2Loader,
+	"pypdf": PyPDFLoader,
+	"pymupdf": PyMuPDFLoader,
+	"unstructuredpdf": UnstructuredPDFLoader,
+	# Common File Types
+	# 1. CSV
+	"csv": CSVLoader,
+	# 2. JSON
+	"json": JSONLoader,
+	# 3. Markdown
+	"unstructuredmarkdown": UnstructuredMarkdownLoader,
+	# 4. HTML
+	"bshtml": BSHTMLLoader,
+	# 5. XML
+	"unstructuredxml": UnstructuredXMLLoader,
+	# 6. All files
+	"directory": DirectoryLoader,
+	"unstructured": UnstructuredLoader,
+	"upstagelayoutanalysis": UpstageLayoutAnalysisLoader,
+}
+
+chunk_modules = {
+	# Llama Index
+	# Token
+	"token": TokenTextSplitter,
+	# Sentence
+	"sentence": SentenceSplitter,
+	# window
+	"sentencewindow": SentenceWindowNodeParser,
+	# Semantic
+	"semantic_llama_index": SemanticSplitterNodeParser,
+	"semanticdoublemerging": SemanticDoubleMergingSplitterNodeParser,
+	# Simple
+	"simplefile": SimpleFileNodeParser,
+	# LangChain
+	# Token
+	"sentencetransformerstoken": SentenceTransformersTokenTextSplitter,
+	# Character
+	"recursivecharacter": RecursiveCharacterTextSplitter,
+	"character": CharacterTextSplitter,
+	# Sentence
+	"konlpy": KonlpyTextSplitter,
+}
+
+
+def split_by_sentence_kiwi() -> Callable[[str], List[str]]:
+	from kiwipiepy import Kiwi
+
+	kiwi = Kiwi()
+
+	def split(text: str) -> List[str]:
+		kiwi_result = kiwi.split_into_sents(text)
+		sentences = list(map(lambda x: x.text, kiwi_result))
+
+		return sentences
+
+	return split
+
+
+sentence_splitter_modules = {"kiwi": LazyInit(split_by_sentence_kiwi)}

autorag-0.2.17/autorag/data/beta/__init__.py

@@ -0,0 +1,3 @@
+# This is v2 version, the next version of data creation
+# The legacy (v1) version will be deprecated on AutoRAG version 0.3
+# The legacy (v1) version and new v2 data creation is not compatible with each other

autorag-0.2.17/autorag/data/beta/extract_evidence.py

@@ -0,0 +1 @@
+# This module is about extracting evidence from the given retrieval gt passage