AutoRAG 0.2.16__tar.gz → 0.2.17__tar.gz

{autorag-0.2.16 → autorag-0.2.17}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.16
+Version: 0.2.17
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License

{autorag-0.2.16 → autorag-0.2.17}/AutoRAG.egg-info/SOURCES.txt

@@ -34,6 +34,7 @@ autorag/validator.py
 autorag/web.py
 autorag/data/__init__.py
 autorag/data/beta/__init__.py
+autorag/data/beta/extract_evidence.py
 autorag/data/beta/sample.py
 autorag/data/beta/schema.py
 autorag/data/beta/filter/__init__.py
@@ -229,9 +230,19 @@ docs/source/data_creation/data_format.md
 docs/source/data_creation/ragas.md
 docs/source/data_creation/tutorial.md
 docs/source/data_creation/beta/data_creation.md
-docs/source/data_creation/beta/filter.md
-docs/source/data_creation/beta/query_gen.md
 docs/source/data_creation/beta/tutorial.md
+docs/source/data_creation/beta/chunk/chunk.md
+docs/source/data_creation/beta/chunk/langchain_chunk.md
+docs/source/data_creation/beta/chunk/llama_index_chunk.md
+docs/source/data_creation/beta/parse/clova.md
+docs/source/data_creation/beta/parse/langchain_parse.md
+docs/source/data_creation/beta/parse/llama_parse.md
+docs/source/data_creation/beta/parse/parse.md
+docs/source/data_creation/beta/parse/table_hybrid_parse.md
+docs/source/data_creation/beta/qa_creation/answer_gen.md
+docs/source/data_creation/beta/qa_creation/filter.md
+docs/source/data_creation/beta/qa_creation/qa_creation.md
+docs/source/data_creation/beta/qa_creation/query_gen.md
 docs/source/deploy/api_endpoint.md
 docs/source/deploy/web.md
 docs/source/evaluate_metrics/generation.md

{autorag-0.2.16 → autorag-0.2.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.2.16
+Version: 0.2.17
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License

autorag-0.2.17/autorag/VERSION

@@ -0,0 +1 @@
+0.2.17

autorag-0.2.17/autorag/data/beta/extract_evidence.py

@@ -0,0 +1 @@
+# This module is about extracting evidence from the given retrieval gt passage

{autorag-0.2.16 → autorag-0.2.17}/autorag/data/beta/schema.py

@@ -1,9 +1,9 @@
 import logging
-from typing import Callable, Optional, Dict, Awaitable, Any
+from typing import Callable, Optional, Dict, Awaitable, Any, Tuple, List
 import pandas as pd
+from autorag.utils.util import process_batch, get_event_loop, fetch_contents
 
 from autorag.support import get_support_modules
-from autorag.utils.util import process_batch, get_event_loop, fetch_contents
 
 logger = logging.getLogger("AutoRAG")
 
@@ -68,6 +68,20 @@ class Corpus:
 	def linked_raw(self, raw: Raw):
 		raise NotImplementedError("linked_raw is read-only.")
 
+	def to_parquet(self, save_path: str):
+		"""
+		Save the corpus to the AutoRAG compatible parquet file.
+		It is not for the data creation, for running AutoRAG.
+		If you want to save it directly, use the below code.
+		`corpus.data.to_parquet(save_path)`
+
+		:param save_path: The path to save the corpus.
+		"""
+		if not save_path.endswith(".parquet"):
+			raise ValueError("save_path must be ended with .parquet")
+		save_df = self.data[["doc_id", "contents", "metadata"]].reset_index(drop=True)
+		save_df.to_parquet(save_path)
+
 	def batch_apply(
 		self, fn: Callable[[Dict, Any], Awaitable[Dict]], batch_size: int = 32, **kwargs
 	) -> "Corpus":
@@ -152,6 +166,26 @@ class QA:
 		)
 		return self
 
+	def to_parquet(self, qa_save_path: str, corpus_save_path: str):
+		"""
+		Save the qa and corpus to the AutoRAG compatible parquet file.
+		It is not for the data creation, for running AutoRAG.
+		If you want to save it directly, use the below code.
+		`qa.data.to_parquet(save_path)`
+
+		:param qa_save_path: The path to save the qa dataset.
+		:param corpus_save_path: The path to save the corpus.
+		"""
+		if not qa_save_path.endswith(".parquet"):
+			raise ValueError("save_path must be ended with .parquet")
+		if not corpus_save_path.endswith(".parquet"):
+			raise ValueError("save_path must be ended with .parquet")
+		save_df = self.data[
+			["qid", "query", "retrieval_gt", "generation_gt"]
+		].reset_index(drop=True)
+		save_df.to_parquet(qa_save_path)
+		self.linked_corpus.to_parquet(corpus_save_path)
+
 	def update_corpus(self, new_corpus: Corpus) -> "QA":
 		"""
 		Update linked corpus.
@@ -163,4 +197,105 @@ class QA:
 		    Must have valid `linked_raw` and `raw_id`, `raw_start_idx`, `raw_end_idx` columns.
 		:return: The QA instance that updated linked corpus.
 		"""
-		pass
+		self.data["evidence_path"] = (
+			self.data["retrieval_gt"]
+			.apply(
+				lambda x: fetch_contents(
+					self.linked_corpus.data,
+					x,
+					column_name="path",
+				)
+			)
+			.tolist()
+		)
+		self.data["evidence_page"] = self.data["retrieval_gt"].apply(
+			lambda x: list(
+				map(
+					lambda lst: list(map(lambda x: x.get("page", -1), lst)),
+					fetch_contents(self.linked_corpus.data, x, column_name="metadata"),
+				)
+			)
+		)
+		if "evidence_start_end_idx" not in self.data.columns:
+			# make evidence start_end_idx
+			self.data["evidence_start_end_idx"] = (
+				self.data["retrieval_gt"]
+				.apply(
+					lambda x: fetch_contents(
+						self.linked_corpus.data,
+						x,
+						column_name="start_end_idx",
+					)
+				)
+				.tolist()
+			)
+
+		# matching the new corpus with the old corpus
+		path_corpus_dict = QA.__make_path_corpus_dict(new_corpus.data)
+		new_retrieval_gt = self.data.apply(
+			lambda row: QA.__match_index_row(
+				row["evidence_start_end_idx"],
+				row["evidence_path"],
+				row["evidence_page"],
+				path_corpus_dict,
+			),
+			axis=1,
+		).tolist()
+		new_qa = self.data.copy(deep=True)[["qid", "query", "generation_gt"]]
+		new_qa["retrieval_gt"] = new_retrieval_gt
+		return QA(new_qa, new_corpus)
+
+	@staticmethod
+	def __match_index(target_idx: Tuple[int, int], dst_idx: Tuple[int, int]) -> bool:
+		"""
+		Check if the target_idx is overlap by the dst_idx.
+		"""
+		target_start, target_end = target_idx
+		dst_start, dst_end = dst_idx
+		return (
+			dst_start <= target_start <= dst_end or dst_start <= target_end <= dst_end
+		)
+
+	@staticmethod
+	def __match_index_row(
+		evidence_indices: List[List[Tuple[int, int]]],
+		evidence_paths: List[List[str]],
+		evidence_pages: List[List[int]],
+		path_corpus_dict: Dict,
+	) -> List[List[str]]:
+		"""
+		Find the matched passage from new_corpus.
+
+		:param evidence_indices: The evidence indices at the corresponding Raw.
+			Its shape is the same as the retrieval_gt.
+		:param evidence_paths: The evidence paths at the corresponding Raw.
+			Its shape is the same as the retrieval_gt.
+		:param path_corpus_dict: The key is the path name, and the value is the corpus dataframe that only contains the path in the key.
+			You can make it using `QA.__make_path_corpus_dict`.
+		:return:
+		"""
+		result = []
+		for i, idx_list in enumerate(evidence_indices):
+			sub_result = []
+			for j, idx in enumerate(idx_list):
+				path_corpus_df = path_corpus_dict[evidence_paths[i][j]]
+				if evidence_pages[i][j] >= 0:
+					path_corpus_df = path_corpus_df.loc[
+						path_corpus_df["metadata"].apply(lambda x: x.get("page", -1))
+						== evidence_pages[i][j]
+					]
+				matched_corpus = path_corpus_df.loc[
+					path_corpus_df["start_end_idx"].apply(
+						lambda x: QA.__match_index(idx, x)
+					)
+				]
+				sub_result.extend(matched_corpus["doc_id"].tolist())
+			result.append(sub_result)
+		return result
+
+	@staticmethod
+	def __make_path_corpus_dict(corpus_df: pd.DataFrame) -> Dict[str, pd.DataFrame]:
+		return {
+			path: corpus_df[corpus_df["path"] == path]
+			for path in corpus_df["path"].unique()
+		}

{autorag-0.2.16 → autorag-0.2.17}/autorag/data/parse/clova.py

@@ -19,6 +19,20 @@ def clova_ocr(
 	batch: int = 8,
 	table_detection: bool = False,
 ) -> Tuple[List[str], List[str], List[int]]:
+	"""
+	Parse documents to use Naver Clova OCR.
+
+	:param data_path_list: The list of data paths to parse.
+	:param url: The URL for Clova OCR.
+		You can get the URL with the guide at https://guide.ncloud-docs.com/docs/clovaocr-example01
+		You can set the environment variable CLOVA_URL, or you can set it directly as a parameter.
+	:param api_key: The API key for Clova OCR.
+		You can get the API key with the guide at https://guide.ncloud-docs.com/docs/clovaocr-example01
+		You can set the environment variable CLOVA_API_KEY, or you can set it directly as a parameter.
+	:param batch: The batch size for parse documents. Default is 8.
+	:param table_detection: Whether to enable table detection. Default is False.
+	:return: tuple of lists containing the parsed texts, path and pages.
+	"""
 	url = os.getenv("CLOVA_URL", None) if url is None else url
 	if url is None:
 		raise KeyError(

{autorag-0.2.16 → autorag-0.2.17}/autorag/data/parse/langchain_parse.py

@@ -9,6 +9,14 @@ from autorag.data.parse.base import parser_node
 def langchain_parse(
 	data_path_list: List[str], parse_method: str, **kwargs
 ) -> Tuple[List[str], List[str], List[int]]:
+	"""
+	Parse documents to use langchain document_loaders(parse) method
+
+	:param data_path_list: The list of data paths to parse.
+	:param parse_method: A langchain document_loaders(parse) method to use.
+	:param kwargs: The extra parameters for creating the langchain document_loaders(parse) instance.
+	:return: tuple of lists containing the parsed texts, path and pages.
+	"""
 	if parse_method in ["directory", "unstructured"]:
 		results = parse_all_files(data_path_list, parse_method, **kwargs)
 		texts, path = results[0], results[1]

{autorag-0.2.16 → autorag-0.2.17}/autorag/data/parse/llamaparse.py

@@ -10,6 +10,16 @@ from autorag.utils.util import process_batch, get_event_loop
 def llama_parse(
 	data_path_list: List[str], batch: int = 8, **kwargs
 ) -> Tuple[List[str], List[str], List[int]]:
+	"""
+	Parse documents to use llama_parse.
+	LLAMA_CLOUD_API_KEY environment variable should be set.
+	You can get the key from https://cloud.llamaindex.ai/api-key
+
+	:param data_path_list: The list of data paths to parse.
+	:param batch: The batch size for parse documents. Default is 8.
+	:param kwargs: The extra parameters for creating the llama_parse instance.
+	:return: tuple of lists containing the parsed texts, path and pages.
+	"""
 	parse_instance = LlamaParse(**kwargs)
 
 	tasks = [

{autorag-0.2.16 → autorag-0.2.17}/autorag/data/parse/table_hybrid_parse.py

@@ -18,6 +18,18 @@ def table_hybrid_parse(
 	table_parse_module: str,
 	table_params: Dict,
 ) -> Tuple[List[str], List[str], List[int]]:
+	"""
+	Parse documents to use table_hybrid_parse method.
+	The table_hybrid_parse method is a hybrid method that combines the parsing results of PDFs with and without tables.
+	It splits the PDF file into pages, separates pages with and without tables, and then parses and merges the results.
+
+	:param data_path_list: The list of data paths to parse.
+	:param text_parse_module: The text parsing module to use. The type should be a string.
+	:param text_params: The extra parameters for the text parsing module. The type should be a dictionary.
+	:param table_parse_module: The table parsing module to use. The type should be a string.
+	:param table_params: The extra parameters for the table parsing module. The type should be a dictionary.
+	:return: tuple of lists containing the parsed texts, path and pages.
+	"""
 	# make save folder directory
 	with tempfile.TemporaryDirectory() as save_dir:
 		text_dir = os.path.join(save_dir, "text")

{autorag-0.2.16 → autorag-0.2.17}/autorag/nodes/retrieval/vectordb.py

@@ -105,14 +105,15 @@ async def vectordb_pure(
 	:param query_embeddings: A list of query embeddings.
 	:param top_k: The number of passages to be retrieved.
 	:param collection: A chroma collection instance that will be used to retrieve passages.
-
-	:return: The tuple contains a list of passage ids that retrieved from vectordb and a list of its scores.
+	:return: The tuple contains a list of passage ids that are retrieved from vectordb and a list of its scores.
 	"""
 	id_result, score_result = [], []
 	for embedded_query in query_embeddings:
 		result = collection.query(query_embeddings=embedded_query, n_results=top_k)
 		id_result.extend(result["ids"])
-		score_result.extend(result["distances"])
+		score_result.extend(
+			list(map(lambda lst: list(map(lambda x: 1 - x, lst)), result["distances"]))
+		)
 
 	# Distribute passages evenly
 	id_result, score_result = evenly_distribute_passages(id_result, score_result, top_k)

autorag-0.2.17/docs/source/data_creation/beta/chunk/chunk.md

@@ -0,0 +1,168 @@
+# Chunk
+
+In this section, we will cover how to chunk parsed result.
+
+It is a crucial step because if the parsed result is not chunked well, the RAG will not be optimized well.
+
+Using only YAML files, you can easily use the various chunk methods.
+The chunked result is saved according to the data format used by AutoRAG.
+
+## Overview
+
+The sample chunk pipeline looks like this.
+
+```python
+from autorag.chunker import Chunker
+
+chunker = Chunker.from_parquet(parsed_data_path="your/parsed/data/path")
+chunker.start_chunking("your/path/to/chunk_config.yaml")
+```
+
+## Features
+
+### 1. Add File Name
+You need to set one of 'English' and 'Korean'
+The 'add_file_name' feature is to add a file_name to chunked_contents.
+This is used to prevent hallucination by retrieving contents from the wrong document.
+Default form of English is `"file_name: {file_name}\n contents: {content}"`
+
+#### Example YAML
+
+```yaml
+modules:
+  - module_type: llama_index_chunk
+    chunk_method: [ Token, Sentence ]
+    chunk_size: [ 1024, 512 ]
+    chunk_overlap: 24
+    add_file_name: english
+```
+
+### 2. Sentence Splitter
+
+The following chunk methods in the `llama_index_chunk` module use the sentence splitter.
+
+- `Semantic_llama_index`
+- `SemanticDoubling`
+- `SentenceWindow`
+
+The following methods use `PunktSentenceTokenizer` as the default sentence splitter.
+
+See below for the available languages of `PunktSentenceTokenizer`.
+
+["Czech, Danish, Dutch, English, Estonian, Finnish, French, German, Greek, Italian, Malayalam, Norwegian, Polish, Portuguese, Russian, Slovenian, Spanish, Swedish, Turkish"]
+
+So if the language you want to use is not in the list, or you want to use a different sentence splitter, you can use the sentence_splitter parameter.
+
+#### Available Sentence Splitter
+- [kiwi](https://github.com/bab2min/kiwipiepy) : For Korean 🇰🇷
+
+#### Example YAML
+
+```yaml
+modules:
+  - module_type: llama_index_chunk
+    chunk_method: [ SentenceWindow ]
+    sentence_splitter: kiwi
+    window_size: 3
+    add_file_name: english
+```
+
+#### Using sentence splitter that is not in the Available Sentence Splitter
+
+If you want to use `kiwi`, you can use the following code.
+
+```python
+from autorag.data import sentence_splitter_modules, LazyInit
+
+def split_by_sentence_kiwi() -> Callable[[str], List[str]]:
+	from kiwipiepy import Kiwi
+
+	kiwi = Kiwi()
+
+	def split(text: str) -> List[str]:
+		kiwi_result = kiwi.split_into_sents(text)
+		sentences = list(map(lambda x: x.text, kiwi_result))
+
+		return sentences
+
+	return split
+
+sentence_splitter_modules["kiwi"] = LazyInit(split_by_sentence_kiwi)
+```
+
+## Run Chunk Pipeline
+
+### 1. Set chunker instance
+
+```python
+from autorag.chunker import Chunker
+
+chunker = Chunker.from_parquet(parsed_data_path="your/parsed/data/path")
+```
+
+```{admonition} Want to specify project folder?
+You can specify project directory with `--project_dir` option or project_dir parameter.
+```
+
+### 2. Set YAML file
+
+Here is an example of how to use the `llama_index_chunk` module.
+
+```yaml
+modules:
+  - module_type: llama_index_chunk
+    chunk_method: [ Token, Sentence ]
+    chunk_size: [ 1024, 512 ]
+    chunk_overlap: 24
+```
+
+### 3. Start chunking
+
+Use `start_chunking` function to start parsing.
+
+```python
+chunker.start_chunking("your/path/to/chunk_config.yaml")
+```
+
+### 4. Check the result
+
+If you set `project_dir` parameter, you can check the result in the project directory.
+If not, you can check the result in the current directory.
+
+The way to check the result is the same as the `Evaluator` and `Parser` in AutoRAG.
+
+A `trial_folder` is created in `project_dir` first.
+
+If the chunking is completed successfully, the following three types of files are created in the trial_folder.
+
+1. Chunked Result
+2. Used YAML file
+3. Summary file
+
+For example, if chunking is performed using three chunk methods, the following files are created.
+`0.parquet`, `1.parquet`, `2.parquet`, `parse_config.yaml`, `summary.csv`
+
+Finally, in the summary.csv file, you can see information about the chunked result, such as what chunk method was used to chunk it.
+
+## Output Columns
+- `doc_id`: Document ID. The type is string.
+- `contents`: The contents of the chunked data. The type is string.
+- `path`: The path of the document. The type is string.
+- `start_end_idx`:
+  - Store index of chunked_str based on original_str before chunking
+  - stored to map the retrieval_gt of Evaluation QA Dataset according to various chunk methods.
+- `metadata`: It is also stored in the passage after the data of the parsed result is chunked. The type is dictionary.
+  - Depending on the dataformat of AutoRAG's `Parsed Result`, metadata should have the following keys: `page`, `last_modified_datetime`, `path`.
+
+#### Supported Chunk Modules
+
+📌 You can check our all Chunk modules
+at [here](https://edai.notion.site/Supporting-Chunk-Modules-8db803dba2ec4cd0a8789659106e86a3?pvs=4)
+
+```{toctree}
+---
+maxdepth: 1
+---
+langchain_chunk.md
+llama_index_chunk.md
+```

autorag-0.2.17/docs/source/data_creation/beta/chunk/langchain_chunk.md

@@ -0,0 +1,47 @@
+# Langchain Chunk
+
+Chunk parsed results to use [langchain text splitters](https://api.python.langchain.com/en/latest/text_splitters_api_reference.html#).
+
+## Available Chunk Method
+
+### 1. Token
+
+- [SentenceTransformersToken](https://api.python.langchain.com/en/latest/sentence_transformers/langchain_text_splitters.sentence_transformers.SentenceTransformersTokenTextSplitter.html)
+
+### 2. Character
+
+- [RecursiveCharacter](https://api.python.langchain.com/en/latest/character/langchain_text_splitters.character.RecursiveCharacterTextSplitter.html)
+- [character](https://api.python.langchain.com/en/latest/character/langchain_text_splitters.character.CharacterTextSplitter.html)
+
+### 3. Sentence
+
+- [konlpy](https://api.python.langchain.com/en/latest/konlpy/langchain_text_splitters.konlpy.KonlpyTextSplitter.html): For Korean 🇰🇷
+
+#### Example YAML
+
+```yaml
+modules:
+  - module_type: langchain_chunk
+    parse_method: konlpy
+    add_file_name: korean
+```
+
+## Using Langchain Chunk Method that is not in the Available Chunk Method
+
+You can find more information about the langchain chunk method at
+[here](https://api.python.langchain.com/en/latest/text_splitters_api_reference.html#)
+
+### How to Use
+
+If you want to use `PythonCodeTextSplitter` that is not in the available chunk method, you can use the following code.
+
+```python
+from autorag.data import chunk_modules
+from langchain.text_splitter import PythonCodeTextSplitter
+
+chunk_modules["python"] = PythonCodeTextSplitter
+```
+
+```{attention}
+The key value in chunk_modules must always be written in lowercase.
+```

autorag-0.2.17/docs/source/data_creation/beta/chunk/llama_index_chunk.md

@@ -0,0 +1,57 @@
+# Llama Index Chunk
+
+Chunk parsed results to use [Llama Index Node_Parsers & Text Splitters](https://docs.llamaindex.ai/en/stable/api_reference/node_parsers/).
+
+## Available Chunk Method
+
+### 1. Token
+
+- [Token](https://docs.llamaindex.ai/en/stable/api_reference/node_parsers/token_text_splitter/)
+
+### 2. Sentence
+
+- [Sentence](https://docs.llamaindex.ai/en/stable/api_reference/node_parsers/sentence_splitter/)
+
+### 3. Window
+
+- [SentenceWindow](https://docs.llamaindex.ai/en/stable/api_reference/node_parsers/sentence_window/)
+
+### 4. Semantic
+
+- [semantic_llama_index](https://docs.llamaindex.ai/en/stable/api_reference/node_parsers/semantic_splitter/)
+- [SemanticDoubleMerging](https://docs.llamaindex.ai/en/stable/examples/node_parsers/semantic_double_merging_chunking/)
+
+### 5. Simple
+
+- [Simple](https://docs.llamaindex.ai/en/v0.10.19/api/llama_index.core.node_parser.SimpleFileNodeParser.html)
+
+#### Example YAML
+
+```yaml
+modules:
+  - module_type: llama_index_chunk
+    chunk_method: [ Token, Sentence ]
+    chunk_size: [ 1024, 512 ]
+    chunk_overlap: 24
+    add_file_name: english
+```
+
+## Using Llama Index Chunk Method that is not in the Available Chunk Method
+
+You can find more information about the llama index chunk method at
+[here](https://docs.llamaindex.ai/en/stable/api_reference/node_parsers/).
+
+### How to Use
+
+If you want to use `HTMLNodeParser` that is not in the available chunk method, you can use the following code.
+
+```python
+from autorag.data import chunk_modules
+from llama_index.core.node_parser import HTMLNodeParser
+
+chunk_modules["html"] = HTMLNodeParser
+```
+
+```{attention}
+The key value in chunk_modules must always be written in lowercase.
+```

{autorag-0.2.16 → autorag-0.2.17}/docs/source/data_creation/beta/data_creation.md

@@ -22,25 +22,14 @@ In this new data creation pipeline, we have three schemas. `Raw`, `QA`, and `Cor
 You can use the corpus to generate the answer for the question.
 You have to make corpus data from your documents using parsing and chunking.
 
-### Functions for data creation
-
-We provide some functions for customizing and running a data creation process.
-Here are the basic concepts of each function.
-
-### `QA` and `Corpus`
-
-- `batch_apply`: Apply the function to each row of the dataset, but run in parallel using `asyncio`.
-You have to use `async` functions for using this function.
-Plus, you can specify the batch size.
-- `map`: You can use this function to do something to its pd.DataFrame value. The input function must be get input of pd.DataFrame and return pd.DataFrame.
-
-
+To see the tutorial of the data creation, check [here](./tutorial.md).
 
 ```{toctree}
 ---
-maxdepth: 1
+maxdepth: 2
 ---
 tutorial.md
-query_gen.md
-filter.md
+qa_creation/qa_creation.md
+chunk/chunk.md
+parse/parse.md
 ```

autorag-0.2.17/docs/source/data_creation/beta/parse/clova.md

@@ -0,0 +1,30 @@
+# Clova
+
+Parse raw documents to use Naver
+[Clova OCR](https://guide.ncloud-docs.com/docs/clovaocr-overview).
+
+Clova OCR divides the document into pages for parsing.
+
+## Table Detection
+
+If you have tables in your raw document, set `table_detection: true` to use clova ocr table detection feature.
+
+### Point
+
+#### 1. HTML Parser
+Clova OCR provides parsed table information in complex JSON format.
+It converts the complex JSON form of the table to HTML for storage in the LLM.
+
+The parser was created by our own AutoRAG team and you can find the detailed code in the `json_to_html_table` function in `autorag.data.parse.clova`.
+
+#### 2. The text information comes separately from the table information.
+If your document is a table + text, the text information comes separately from the table information.
+So when using table_detection, it will be saved in `{text}\n\ntable html:\n{table}` format.
+
+## Example YAML
+
+```yaml
+modules:
+  - module_type: clova
+    table_detection: true
+```