AutoRAG 0.3.7__tar.gz → 0.3.8__tar.gz

{autorag-0.3.7 → autorag-0.3.8}/.github/workflows/docker-push.yml

@@ -67,20 +67,20 @@ jobs:
       uses: docker/build-push-action@v6.9.0
       with:
         context: .
-        file: ./docker/base/Dockerfile
+        file: Dockerfile.base
         push: true
         # push: ${{ github.event_name != 'pull_request' && matrix.variant != 'test' }}
-        tags: ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
+        tags: ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
           TARGET_STAGE=${{ matrix.variant }}
 
     - name: Tag and push 'all' for production
       run: |
-        docker pull ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
-        docker tag ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}} ${{ env.DOCKER_REPO }}:latest
-        docker push ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
-        docker push ${{ env.DOCKER_REPO }}:latest
+        docker pull ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }}
+        docker tag ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }} ${{ env.DOCKER_REPO }}:${{matrix.variant}}-latest
+        docker push ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }}
+        docker push ${{ env.DOCKER_REPO }}:${{matrix.variant}}-latest
 
     # - name: Update Docker Hub description
     #   uses: peter-evans/dockerhub-description@v3

{autorag-0.3.7 → autorag-0.3.8}/.github/workflows/gpu-docker-push.yml

@@ -67,20 +67,20 @@ jobs:
       uses: docker/build-push-action@v6.9.0
       with:
         context: .
-        file: ./docker/gpu/Dockerfile
+        file: ./Dockerfile.gpu
         push: true
         # push: ${{ github.event_name != 'pull_request' && matrix.variant != 'test' }}
-        tags: ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
+        tags: ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
           TARGET_STAGE=${{ matrix.variant }}
 
     - name: Tag and push 'all' for production
       run: |
-        docker pull ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
-        docker tag ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}} ${{ env.DOCKER_REPO }}:latest
-        docker push ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
-        docker push ${{ env.DOCKER_REPO }}:latest
+        docker pull ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }}
+        docker tag ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }} ${{ env.DOCKER_REPO }}:${{matrix.variant}}-latest
+        docker push ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }}
+        docker push ${{ env.DOCKER_REPO }}:${{matrix.variant}}-latest
 
     # - name: Update Docker Hub description
     #   uses: peter-evans/dockerhub-description@v3

{autorag-0.3.7 → autorag-0.3.8}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.3.7
+Version: 0.3.8
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -232,7 +232,6 @@ Requires-Dist: sacrebleu
 Requires-Dist: evaluate
 Requires-Dist: rouge_score
 Requires-Dist: rich
-Requires-Dist: chromadb>=0.5.0
 Requires-Dist: click
 Requires-Dist: cohere>=5.8.0
 Requires-Dist: tokenlog>=0.0.2
@@ -242,7 +241,9 @@ Requires-Dist: mixedbread-ai
 Requires-Dist: llama-index-llms-bedrock
 Requires-Dist: scikit-learn
 Requires-Dist: pymilvus
+Requires-Dist: chromadb>=0.5.0
 Requires-Dist: quart
+Requires-Dist: pyngrok
 Requires-Dist: llama-index>=0.11.0
 Requires-Dist: llama-index-core>=0.11.0
 Requires-Dist: llama-index-readers-file
@@ -332,6 +333,16 @@ Plus, join our 📞 [Discord](https://discord.gg/P4DYXfmSAs) Community.
 
 ---
 
+Do you have any difficulties in optimizing your RAG pipeline?
+Or is it hard to set up things to use AutoRAG?
+Try [**AutoRAG Cloud**](https://tally.so/r/n0jOrZ) beta.
+We will help you to run AutoRAG and optimize.
+Plus, we can help you to build RAG evaluation dataset.
+
+Starts with 9.99$ per optimization.
+
+---
+
 ## YouTube Tutorial
 
 https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/c0d23896-40c0-479f-a17b-aa2ec3183a26
@@ -701,7 +712,7 @@ runner.run('your question')
 
 You can run this pipeline as an API server.
 
-Check out the API endpoint at [here](deploy/api_endpoint.md).
+Check out the API endpoint at [here](./docs/source/deploy/api_endpoint.md).
 
 ```python
 import nest_asyncio
@@ -733,6 +744,15 @@ autorag run_web --trial_path your/path/to/trial_path
 
 <img width="1491" alt="web_interface" src="https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/f6b00353-f6bb-4d8f-8740-1c264c0acbb8">
 
+### Use advanced web interface
+
+You can deploy the advanced web interface featured by [Kotaemon](https://github.com/Cinnamon/kotaemon) to the fly.io.
+Go [here](https://github.com/vkehfdl1/AutoRAG-web-kotaemon) to use it and deploy to the fly.io.
+
+Example :
+
+![Kotaemon Example](https://velog.velcdn.com/images/autorag/post/5e71b8d9-3e59-4e63-9191-355a1a5aa3a0/image.png)
+
 ## 📌 Supporting Data Creation Modules
 ![Data Creation](https://github.com/user-attachments/assets/0e5872de-2892-46b4-9ecd-e395671e324c)
 

{autorag-0.3.7 → autorag-0.3.8}/AutoRAG.egg-info/SOURCES.txt

@@ -3,6 +3,8 @@
 .pre-commit-config.yaml
 CODE_OF_CONDUCT.md
 CONTRIBUTING.md
+Dockerfile.base
+Dockerfile.gpu
 LICENSE
 README.md
 build_and_push.sh
@@ -186,8 +188,6 @@ autorag/vectordb/__init__.py
 autorag/vectordb/base.py
 autorag/vectordb/chroma.py
 autorag/vectordb/milvus.py
-docker/base/Dockerfile
-docker/gpu/Dockerfile
 docs/Makefile
 docs/make.bat
 docs/requirements.txt
@@ -201,6 +201,7 @@ docs/source/structure.md
 docs/source/troubleshooting.md
 docs/source/tutorial.md
 docs/source/_static/data_creation.png
+docs/source/_static/data_creation_pipeline.png
 docs/source/_static/data_folder.png
 docs/source/_static/dcg.png
 docs/source/_static/f1_score.png
@@ -382,6 +383,7 @@ sample_dataset/eli5/load_eli5_dataset.py
 sample_dataset/hotpotqa/load_hotpotqa_dataset.py
 sample_dataset/msmarco/load_msmarco_dataset.py
 sample_dataset/triviaqa/load_triviaqa_dataset.py
+tests/__init__.py
 tests/conftest.py
 tests/delete_tests.py
 tests/mock.py
@@ -623,4 +625,5 @@ tests/resources/result_project/resources/chroma/6595d304-5270-4d9f-a267-c1913668
 tests/resources/sample_project/data/corpus.parquet
 tests/resources/sample_project/data/qa.parquet
 tests/resources/sample_project/resources/bm25_gpt2.pkl
-tests/resources/sample_project/resources/bm25_porter_stemmer.pkl
+tests/resources/sample_project/resources/bm25_porter_stemmer.pkl
+tests/resources/sample_project/resources/vectordb.yaml

{autorag-0.3.7 → autorag-0.3.8}/AutoRAG.egg-info/requires.txt

@@ -11,7 +11,6 @@ sacrebleu
 evaluate
 rouge_score
 rich
-chromadb>=0.5.0
 click
 cohere>=5.8.0
 tokenlog>=0.0.2
@@ -21,7 +20,9 @@ mixedbread-ai
 llama-index-llms-bedrock
 scikit-learn
 pymilvus
+chromadb>=0.5.0
 quart
+pyngrok
 llama-index>=0.11.0
 llama-index-core>=0.11.0
 llama-index-readers-file

autorag-0.3.8/Dockerfile.base

@@ -0,0 +1,71 @@
+# Base stage: Install common dependencies
+FROM python:3.10-slim AS base
+
+# Set working directory and environment variables
+WORKDIR /usr/src/app
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1
+
+# Copy only requirements files first to leverage Docker cache
+COPY pyproject.toml ./
+
+# Install system and Python dependencies in a single layer
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    build-essential \
+    gcc \
+    libssl-dev && \
+    pip install --upgrade pip setuptools setuptools-scm && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /root/.cache/pip/*
+
+# Copy project files
+COPY . .
+
+# Install base project
+RUN pip install -e ./ && \
+    rm -rf /root/.cache/pip/*
+
+# Ko stage with minimal dependencies
+FROM base AS ko
+RUN pip install --no-cache-dir "AutoRAG[ko]" && \
+    rm -rf /root/.cache/pip/*
+ENTRYPOINT ["python", "-m", "autorag.cli"]
+
+# Dev stage with all development tools
+FROM base AS dev
+RUN pip install --no-cache-dir \
+    -r ./tests/requirements.txt \
+    -r ./docs/requirements.txt \
+    "AutoRAG[dev]" && \
+    rm -rf /root/.cache/pip/*
+ENTRYPOINT ["python", "-m", "autorag.cli"]
+
+# Ja stage with Japanese support
+FROM base AS ja
+RUN pip install --no-cache-dir "AutoRAG[ja]" && \
+    rm -rf /root/.cache/pip/*
+ENTRYPOINT ["python", "-m", "autorag.cli"]
+
+# API stage with minimal footprint
+FROM base AS api
+RUN rm -rf \
+    ./sample_dataset \
+    ./tests \
+    ./docs \
+    /root/.cache/pip/*
+ENTRYPOINT ["python", "-m", "autorag.cli"]
+
+# Parsing stage with OCR support
+FROM base AS parsing
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    poppler-utils \
+    tesseract-ocr \
+    tesseract-ocr-eng \
+    tesseract-ocr-kor && \
+    pip install --no-cache-dir "AutoRAG[parse]" && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /root/.cache/pip/*
+ENTRYPOINT ["python", "-m", "autorag.cli"]

{autorag-0.3.7 → autorag-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.3.7
+Version: 0.3.8
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -232,7 +232,6 @@ Requires-Dist: sacrebleu
 Requires-Dist: evaluate
 Requires-Dist: rouge_score
 Requires-Dist: rich
-Requires-Dist: chromadb>=0.5.0
 Requires-Dist: click
 Requires-Dist: cohere>=5.8.0
 Requires-Dist: tokenlog>=0.0.2
@@ -242,7 +241,9 @@ Requires-Dist: mixedbread-ai
 Requires-Dist: llama-index-llms-bedrock
 Requires-Dist: scikit-learn
 Requires-Dist: pymilvus
+Requires-Dist: chromadb>=0.5.0
 Requires-Dist: quart
+Requires-Dist: pyngrok
 Requires-Dist: llama-index>=0.11.0
 Requires-Dist: llama-index-core>=0.11.0
 Requires-Dist: llama-index-readers-file
@@ -332,6 +333,16 @@ Plus, join our 📞 [Discord](https://discord.gg/P4DYXfmSAs) Community.
 
 ---
 
+Do you have any difficulties in optimizing your RAG pipeline?
+Or is it hard to set up things to use AutoRAG?
+Try [**AutoRAG Cloud**](https://tally.so/r/n0jOrZ) beta.
+We will help you to run AutoRAG and optimize.
+Plus, we can help you to build RAG evaluation dataset.
+
+Starts with 9.99$ per optimization.
+
+---
+
 ## YouTube Tutorial
 
 https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/c0d23896-40c0-479f-a17b-aa2ec3183a26
@@ -701,7 +712,7 @@ runner.run('your question')
 
 You can run this pipeline as an API server.
 
-Check out the API endpoint at [here](deploy/api_endpoint.md).
+Check out the API endpoint at [here](./docs/source/deploy/api_endpoint.md).
 
 ```python
 import nest_asyncio
@@ -733,6 +744,15 @@ autorag run_web --trial_path your/path/to/trial_path
 
 <img width="1491" alt="web_interface" src="https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/f6b00353-f6bb-4d8f-8740-1c264c0acbb8">
 
+### Use advanced web interface
+
+You can deploy the advanced web interface featured by [Kotaemon](https://github.com/Cinnamon/kotaemon) to the fly.io.
+Go [here](https://github.com/vkehfdl1/AutoRAG-web-kotaemon) to use it and deploy to the fly.io.
+
+Example :
+
+![Kotaemon Example](https://velog.velcdn.com/images/autorag/post/5e71b8d9-3e59-4e63-9191-355a1a5aa3a0/image.png)
+
 ## 📌 Supporting Data Creation Modules
 ![Data Creation](https://github.com/user-attachments/assets/0e5872de-2892-46b4-9ecd-e395671e324c)
 

{autorag-0.3.7 → autorag-0.3.8}/README.md

@@ -30,6 +30,16 @@ Plus, join our 📞 [Discord](https://discord.gg/P4DYXfmSAs) Community.
 
 ---
 
+Do you have any difficulties in optimizing your RAG pipeline?
+Or is it hard to set up things to use AutoRAG?
+Try [**AutoRAG Cloud**](https://tally.so/r/n0jOrZ) beta.
+We will help you to run AutoRAG and optimize.
+Plus, we can help you to build RAG evaluation dataset.
+
+Starts with 9.99$ per optimization.
+
+---
+
 ## YouTube Tutorial
 
 https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/c0d23896-40c0-479f-a17b-aa2ec3183a26
@@ -399,7 +409,7 @@ runner.run('your question')
 
 You can run this pipeline as an API server.
 
-Check out the API endpoint at [here](deploy/api_endpoint.md).
+Check out the API endpoint at [here](./docs/source/deploy/api_endpoint.md).
 
 ```python
 import nest_asyncio
@@ -431,6 +441,15 @@ autorag run_web --trial_path your/path/to/trial_path
 
 <img width="1491" alt="web_interface" src="https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/f6b00353-f6bb-4d8f-8740-1c264c0acbb8">
 
+### Use advanced web interface
+
+You can deploy the advanced web interface featured by [Kotaemon](https://github.com/Cinnamon/kotaemon) to the fly.io.
+Go [here](https://github.com/vkehfdl1/AutoRAG-web-kotaemon) to use it and deploy to the fly.io.
+
+Example :
+
+![Kotaemon Example](https://velog.velcdn.com/images/autorag/post/5e71b8d9-3e59-4e63-9191-355a1a5aa3a0/image.png)
+
 ## 📌 Supporting Data Creation Modules
 ![Data Creation](https://github.com/user-attachments/assets/0e5872de-2892-46b4-9ecd-e395671e324c)
 

autorag-0.3.8/autorag/VERSION

@@ -0,0 +1 @@
+0.3.8

{autorag-0.3.7 → autorag-0.3.8}/autorag/chunker.py

@@ -1,8 +1,6 @@
-import json
 import logging
 import os
 import shutil
-from datetime import datetime
 from typing import Optional
 
 import pandas as pd
@@ -32,13 +30,11 @@ class Chunker:
 		return cls(parsed_result, project_dir)
 
 	def start_chunking(self, yaml_path: str):
-		trial_name = self.__get_new_trial_name()
-		self.__make_trial_dir(trial_name)
+		if not os.path.exists(self.project_dir):
+			os.makedirs(self.project_dir)
 
 		# Copy YAML file to the trial directory
-		shutil.copy(
-			yaml_path, os.path.join(self.project_dir, trial_name, "chunk_config.yaml")
-		)
+		shutil.copy(yaml_path, os.path.join(self.project_dir, "chunk_config.yaml"))
 
 		# load yaml file
 		modules = load_yaml(yaml_path)
@@ -50,32 +46,6 @@ class Chunker:
 			modules=input_modules,
 			module_params=input_params,
 			parsed_result=self.parsed_raw,
-			trial_path=os.path.join(self.project_dir, trial_name),
+			project_dir=self.project_dir,
 		)
 		logger.info("Chunking Done!")
-
-	def __get_new_trial_name(self) -> str:
-		trial_json_path = os.path.join(self.project_dir, "trial.json")
-		if not os.path.exists(trial_json_path):
-			return "0"
-		with open(trial_json_path, "r") as f:
-			trial_json = json.load(f)
-		return str(int(trial_json[-1]["trial_name"]) + 1)
-
-	def __make_trial_dir(self, trial_name: str):
-		trial_json_path = os.path.join(self.project_dir, "trial.json")
-		if os.path.exists(trial_json_path):
-			with open(trial_json_path, "r") as f:
-				trial_json = json.load(f)
-		else:
-			trial_json = []
-
-		trial_json.append(
-			{
-				"trial_name": trial_name,
-				"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-			}
-		)
-		os.makedirs(os.path.join(self.project_dir, trial_name))
-		with open(trial_json_path, "w") as f:
-			json.dump(trial_json, f, indent=4)

{autorag-0.3.7 → autorag-0.3.8}/autorag/cli.py

@@ -47,7 +47,7 @@ def cli():
 )
 def evaluate(config, qa_data_path, corpus_data_path, project_dir, skip_validation):
 	if not config.endswith(".yaml") and not config.endswith(".yml"):
-		raise ValueError(f"Config file {config} is not a parquet file.")
+		raise ValueError(f"Config file {config} is not a yaml or yml file.")
 	if not os.path.exists(config):
 		raise ValueError(f"Config file {config} does not exist.")
 	evaluator = Evaluator(qa_data_path, corpus_data_path, project_dir=project_dir)

{autorag-0.3.7 → autorag-0.3.8}/autorag/data/chunk/base.py

@@ -115,8 +115,8 @@ def add_file_name(
 				zip(chunk_texts, file_names),
 			)
 		)
-	elif file_name_language == 'ja':
-				return list(
+	elif file_name_language == "ja":
+		return list(
 			map(
 				lambda x: f"ファイル名: {x[1]}\n 内容: {x[0]}",
 				zip(chunk_texts, file_names),

{autorag-0.3.7 → autorag-0.3.8}/autorag/data/chunk/run.py

@@ -9,7 +9,7 @@ def run_chunker(
 	modules: List[Callable],
 	module_params: List[Dict],
 	parsed_result: pd.DataFrame,
-	trial_path: str,
+	project_dir: str,
 ):
 	results, execution_times = zip(
 		*map(
@@ -21,7 +21,7 @@ def run_chunker(
 
 	# save results to parquet files
 	filepaths = list(
-		map(lambda x: os.path.join(trial_path, f"{x}.parquet"), range(len(modules)))
+		map(lambda x: os.path.join(project_dir, f"{x}.parquet"), range(len(modules)))
 	)
 	list(map(lambda x: x[0].to_parquet(x[1], index=False), zip(results, filepaths)))
 	filenames = list(map(lambda x: os.path.basename(x), filepaths))
@@ -34,5 +34,5 @@ def run_chunker(
 			"execution_time": average_times,
 		}
 	)
-	summary_df.to_csv(os.path.join(trial_path, "summary.csv"), index=False)
+	summary_df.to_csv(os.path.join(project_dir, "summary.csv"), index=False)
 	return summary_df

{autorag-0.3.7 → autorag-0.3.8}/autorag/data/parse/run.py

@@ -9,7 +9,7 @@ def run_parser(
 	modules: List[Callable],
 	module_params: List[Dict],
 	data_path_glob: str,
-	trial_path: str,
+	project_dir: str,
 ):
 	results, execution_times = zip(
 		*map(
@@ -21,7 +21,7 @@ def run_parser(
 
 	# save results to parquet files
 	filepaths = list(
-		map(lambda x: os.path.join(trial_path, f"{x}.parquet"), range(len(modules)))
+		map(lambda x: os.path.join(project_dir, f"{x}.parquet"), range(len(modules)))
 	)
 	list(map(lambda x: x[0].to_parquet(x[1], index=False), zip(results, filepaths)))
 	filenames = list(map(lambda x: os.path.basename(x), filepaths))
@@ -34,5 +34,5 @@ def run_parser(
 			"execution_time": average_times,
 		}
 	)
-	summary_df.to_csv(os.path.join(trial_path, "summary.csv"), index=False)
+	summary_df.to_csv(os.path.join(project_dir, "summary.csv"), index=False)
 	return summary_df

{autorag-0.3.7 → autorag-0.3.8}/autorag/data/qa/filter/dontknow.py

@@ -36,8 +36,8 @@ dont_know_phrases = {
 		"よく分かってません",
 		"わかりかねます",
 		"存じません",
-		"お答えいたしかねます"
-	]
+		"お答えいたしかねます",
+	],
 }
 
 

{autorag-0.3.7 → autorag-0.3.8}/autorag/data/qa/filter/prompt.py

@@ -16,13 +16,13 @@ If the answer implies 'I don't know', return True. If not, return False.""",
 만약 해당 문장이 '모른다고' 답한 것이라면, True를 반환하세요. 그렇지 않다면 False를 반환하세요.""",
 			)
 		],
-        "ja":[
-            ChatMessage(
+		"ja": [
+			ChatMessage(
 				role=MessageRole.SYSTEM,
 				content="""次の文章はある質問に対する答えです。 該当文章が質問に対して「知らない」と答えたのか判断します。
 もし、その文章が「知らない」と答えたのであれば、Trueを返します。 そうでなければFalseを返します。""",
 			)
-		]
+		],
 	},
 	"passage_dependency": {
 		"en": [
@@ -55,8 +55,8 @@ Please return True if the input question is passage dependent. Else return False
 입력된 질문이 단락 의존적이라면 True를 반환하고, 그렇지 않으면 False를 반환하세요.""",
 			)
 		],
-        "ja":[
-            ChatMessage(
+		"ja": [
+			ChatMessage(
 				role=MessageRole.SYSTEM,
 				content="""あなたは「段落依存」の質問を認識する分類器です。
 「段落依存」とは、どの段落が選択されるかによって答えが変わる質問を意味します。
@@ -68,6 +68,6 @@ KBO野球史上1試合で最も高い点数は何ですか?またはフランス
 
 入力された質問が段落依存的である場合はTrueを返し、そうでない場合はFalseを返します。""",
 			)
-		]
+		],
 	},
 }

{autorag-0.3.7 → autorag-0.3.8}/autorag/data/qa/generation_gt/llama_index_gen_gt.py

@@ -9,7 +9,7 @@ from autorag.data.qa.generation_gt.base import add_gen_gt
 from autorag.data.qa.generation_gt.prompt import GEN_GT_SYSTEM_PROMPT
 
 
-async def make_gen_gt_llama_index(row: Dict, llm: BaseLLM, system_prompt: str):
+async def make_gen_gt_llama_index(row: Dict, llm: BaseLLM, system_prompt: str) -> Dict:
 	retrieval_gt_contents = list(
 		itertools.chain.from_iterable(row["retrieval_gt_contents"])
 	)
@@ -35,3 +35,7 @@ async def make_concise_gen_gt(row: Dict, llm: BaseLLM, lang: str = "en") -> Dict
 
 async def make_basic_gen_gt(row: Dict, llm: BaseLLM, lang: str = "en") -> Dict:
 	return await make_gen_gt_llama_index(row, llm, GEN_GT_SYSTEM_PROMPT["basic"][lang])
+
+
+async def make_custom_gen_gt(row: Dict, llm: BaseLLM, system_prompt: str) -> Dict:
+	return await make_gen_gt_llama_index(row, llm, system_prompt)

{autorag-0.3.7 → autorag-0.3.8}/autorag/data/qa/generation_gt/prompt.py

@@ -10,18 +10,18 @@ It doesn't have to be an full sentence. It can be the answer is a word or a para
 답변은 간결하고 질문에 관련된 내용만 포함해야 합니다.
 불필요하게 길게 답변하지 말고, 명확하게 작성하세요.
 완전한 문장이 아니어도 되며, 답은 단어나 요약일 수 있습니다.""",
-		"ja":"""
+		"ja": """
 あなたは与えられた質問に対して提供されたText内で答えを探すAI秘書です。
 質問に対する答えをTextで探して適切な答えを作成しましょう。
 回答は簡潔で、質問に関連する内容のみを含める必要があります。
 不必要に長く答えず、明確に作成しましょう。
 完全な文章でなくてもいいし、答えは単語や要約かもしれません。
-"""
+""",
 	},
 	"basic": {
 		"en": """You are an AI assistant to answer the given question in the provide evidence text.
 You can find the evidence from the given text about question, and you have to write a proper answer to the given question.""",
 		"ko": "당신은 주어진 질문에 대한 답을 제공된 Text 내에서 찾는 AI 비서입니다. 질문과 관련된 증거를 Text에서 찾아 적절한 답변을 작성하세요.",
-        "ja": "あなたは与えられた質問に対する答えを提供されたText内で探すAI秘書です。 質問に関する証拠をTextで探して適切な回答を作成しましょう。"
+		"ja": "あなたは与えられた質問に対する答えを提供されたText内で探すAI秘書です。 質問に関する証拠をTextで探して適切な回答を作成しましょう。",
 	},
 }

{autorag-0.3.7 → autorag-0.3.8}/autorag/data/qa/query/llama_gen_query.py

@@ -13,13 +13,11 @@ async def llama_index_generate_base(
 	messages: List[ChatMessage],
 ) -> Dict:
 	context = list(itertools.chain.from_iterable(row["retrieval_gt_contents"]))
-	context_str = "Text:\n" + "\n".join(
-		[f"{i + 1}. {c}" for i, c in enumerate(context)]
-	)
+	context_str = "\n".join([f"{i + 1}. {c}" for i, c in enumerate(context)])
 	user_prompt = f"Text:\n{context_str}\n\nGenerated Question from the Text:\n"
-	messages.append(ChatMessage(role=MessageRole.USER, content=user_prompt))
-
-	chat_response: ChatResponse = await llm.achat(messages=messages)
+	user_message = ChatMessage(role=MessageRole.USER, content=user_prompt)
+	new_messages = [*messages, user_message]
+	chat_response: ChatResponse = await llm.achat(messages=new_messages)
 	row["query"] = chat_response.message.content
 	return row
 
@@ -62,3 +60,11 @@ async def two_hop_incremental(
 	response = chat_response.message.content
 	row["query"] = response.split(":")[-1].strip()
 	return row
+
+
+async def custom_query_gen(
+	row: Dict,
+	llm: BaseLLM,
+	messages: List[ChatMessage],
+) -> Dict:
+	return await llama_index_generate_base(row, llm, messages)