AutoRAG 0.3.3__tar.gz → 0.3.4__tar.gz

autorag-0.3.4/.github/workflows/docker-push.yml

@@ -0,0 +1,91 @@
+name: Docker Push
+
+on:
+  push:
+    branches: [ "main" ]
+
+env:
+  DOCKER_REPO: "autoraghq/autorag"
+
+jobs:
+  check-version:
+    runs-on: ubuntu-latest
+    outputs:
+      changed: ${{ steps.version_changed.outputs.changed }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4.2.0
+
+    - name: Get changed files
+      id: changed-files
+      uses: tj-actions/changed-files@v45.0.3
+
+    - name: Check for VERSION file change
+      id: version_changed
+      env:
+        ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+      run: |
+        echo "changed=false" >> $GITHUB_OUTPUT
+        if echo "${ALL_CHANGED_FILES}" | grep -q 'VERSION'; then
+          echo "changed=true" >> $GITHUB_OUTPUT
+        fi
+
+  build-and-push:
+    needs: check-version
+    if: needs.check-version.outputs.changed == 'true'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        variant: [all] # [all, ko, dev, parsing]
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4.2.0
+
+    - name: Read VERSION file
+      run: echo "VERSION=$(cat autorag/VERSION)" >> $GITHUB_ENV
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3.7.1
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v3.3.0
+      with:
+        username: ${{ secrets.DOCKER_HUB_USERNAME }}
+        password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+    - name: Extract metadata (tags, labels) for Docker
+      id: meta
+      uses: docker/metadata-action@v5.5.1
+      with:
+        images: ${{ env.DOCKER_REPO }}
+        tags: |
+          type=raw,value=${{ env.VERSION }}-${{ matrix.variant }}
+          type=raw,value=${{ matrix.variant }},enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
+          type=raw,value=latest-${{ matrix.variant }},enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
+
+    - name: Build and push Docker image
+      uses: docker/build-push-action@v6.9.0
+      with:
+        context: .
+        file: ./Dockerfile
+        push: true
+        # push: ${{ github.event_name != 'pull_request' && matrix.variant != 'test' }}
+        tags: ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
+        labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          TARGET_STAGE=${{ matrix.variant }}
+
+    - name: Tag and push 'all' for production
+      run: |
+        docker pull ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
+        docker tag ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}} ${{ env.DOCKER_REPO }}:latest
+        docker push ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
+        docker push ${{ env.DOCKER_REPO }}:latest
+
+    # - name: Update Docker Hub description
+    #   uses: peter-evans/dockerhub-description@v3
+    #   with:
+    #     username: ${{ secrets.DOCKER_HUB_USERNAME }}
+    #     password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+    #     repository: ${{ env.DOCKER_REPO }}
+    #     short-description: ${{ github.event.repository.description }}

{autorag-0.3.3 → autorag-0.3.4}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.3.3
+Version: 0.3.4
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -235,7 +235,6 @@ Requires-Dist: rouge_score
 Requires-Dist: rich
 Requires-Dist: chromadb>=0.5.0
 Requires-Dist: click
-Requires-Dist: Flask
 Requires-Dist: torch
 Requires-Dist: sentencepiece
 Requires-Dist: cohere>=5.8.0
@@ -246,6 +245,10 @@ Requires-Dist: sentence-transformers
 Requires-Dist: FlagEmbedding
 Requires-Dist: llmlingua
 Requires-Dist: peft
+Requires-Dist: optimum[nncf,openvino]
+Requires-Dist: voyageai
+Requires-Dist: mixedbread-ai
+Requires-Dist: quart
 Requires-Dist: llama-index>=0.11.0
 Requires-Dist: llama-index-core>=0.11.0
 Requires-Dist: llama-index-readers-file
@@ -282,10 +285,14 @@ Requires-Dist: jq; extra == "parse"
 Requires-Dist: unstructured[pdf]; extra == "parse"
 Requires-Dist: PyPDF2<3.0; extra == "parse"
 Requires-Dist: pdf2image; extra == "parse"
+Provides-Extra: ja
+Requires-Dist: sudachipy>=0.6.8; extra == "ja"
+Requires-Dist: sudachidict_core; extra == "ja"
 Provides-Extra: all
 Requires-Dist: AutoRAG[ko]; extra == "all"
 Requires-Dist: AutoRAG[dev]; extra == "all"
 Requires-Dist: AutoRAG[parse]; extra == "all"
+Requires-Dist: AutoRAG[ja]; extra == "all"
 
 # AutoRAG
 
@@ -655,14 +662,17 @@ You can run this pipeline as an API server.
 Check out the API endpoint at [here](deploy/api_endpoint.md).
 
 ```python
-from autorag.deploy import Runner
+import nest_asyncio
+from autorag.deploy import ApiRunner
 
-runner = Runner.from_trial_folder('/your/path/to/trial_dir')
+nest_asyncio.apply()
+
+runner = ApiRunner.from_trial_folder('/your/path/to/trial_dir')
 runner.run_api_server()
 ```
 
 ```bash
-autorag run_api --config_path your/path/to/pipeline.yaml --host 0.0.0.0 --port 8000
+autorag run_api --trial_dir your/path/to/trial_dir --host 0.0.0.0 --port 8000
 ```
 
 The cli command uses extracted config YAML file. If you want to know it more, check out [here](https://docs.auto-rag.com/tutorial.html#extract-pipeline-and-evaluate-test-dataset).

{autorag-0.3.3 → autorag-0.3.4}/AutoRAG.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ CONTRIBUTING.md
 Dockerfile
 LICENSE
 README.md
+build_and_push.sh
 docker-compose.yml
 pyproject.toml
 requirements.txt
@@ -13,6 +14,7 @@ requirements.txt
 .github/dependabot.yml
 .github/ISSUE_TEMPLATE/bug_report.md
 .github/ISSUE_TEMPLATE/feature_request.md
+.github/workflows/docker-push.yml
 .github/workflows/publish.yml
 .github/workflows/sphinx.yml
 .github/workflows/test.yml
@@ -27,7 +29,6 @@ autorag/__init__.py
 autorag/chunker.py
 autorag/cli.py
 autorag/dashboard.py
-autorag/deploy.py
 autorag/evaluator.py
 autorag/node_line.py
 autorag/parser.py
@@ -81,6 +82,11 @@ autorag/data/qa/query/openai_gen_query.py
 autorag/data/qa/query/prompt.py
 autorag/data/utils/__init__.py
 autorag/data/utils/util.py
+autorag/deploy/__init__.py
+autorag/deploy/api.py
+autorag/deploy/base.py
+autorag/deploy/gradio.py
+autorag/deploy/swagger.yml
 autorag/evaluation/__init__.py
 autorag/evaluation/generation.py
 autorag/evaluation/retrieval.py
@@ -133,13 +139,16 @@ autorag/nodes/passagereranker/flag_embedding.py
 autorag/nodes/passagereranker/flag_embedding_llm.py
 autorag/nodes/passagereranker/jina.py
 autorag/nodes/passagereranker/koreranker.py
+autorag/nodes/passagereranker/mixedbreadai.py
 autorag/nodes/passagereranker/monot5.py
+autorag/nodes/passagereranker/openvino.py
 autorag/nodes/passagereranker/pass_reranker.py
 autorag/nodes/passagereranker/rankgpt.py
 autorag/nodes/passagereranker/run.py
 autorag/nodes/passagereranker/sentence_transformer.py
 autorag/nodes/passagereranker/time_reranker.py
 autorag/nodes/passagereranker/upr.py
+autorag/nodes/passagereranker/voyageai.py
 autorag/nodes/passagereranker/tart/__init__.py
 autorag/nodes/passagereranker/tart/modeling_enc_t5.py
 autorag/nodes/passagereranker/tart/tart.py
@@ -228,6 +237,7 @@ docs/source/api_spec/autorag.data.qa.rst
 docs/source/api_spec/autorag.data.qacreation.rst
 docs/source/api_spec/autorag.data.rst
 docs/source/api_spec/autorag.data.utils.rst
+docs/source/api_spec/autorag.deploy.rst
 docs/source/api_spec/autorag.evaluation.metric.rst
 docs/source/api_spec/autorag.evaluation.rst
 docs/source/api_spec/autorag.nodes.generator.rst
@@ -291,13 +301,16 @@ docs/source/nodes/passage_reranker/flag_embedding_llm_reranker.md
 docs/source/nodes/passage_reranker/flag_embedding_reranker.md
 docs/source/nodes/passage_reranker/jina_reranker.md
 docs/source/nodes/passage_reranker/koreranker.md
+docs/source/nodes/passage_reranker/mixedbreadai_reranker.md
 docs/source/nodes/passage_reranker/monot5.md
+docs/source/nodes/passage_reranker/openvino_reranker.md
 docs/source/nodes/passage_reranker/passage_reranker.md
 docs/source/nodes/passage_reranker/rankgpt.md
 docs/source/nodes/passage_reranker/sentence_transformer_reranker.md
 docs/source/nodes/passage_reranker/tart.md
 docs/source/nodes/passage_reranker/time_reranker.md
 docs/source/nodes/passage_reranker/upr.md
+docs/source/nodes/passage_reranker/voyageai_reranker.md
 docs/source/nodes/prompt_maker/fstring.md
 docs/source/nodes/prompt_maker/long_context_reorder.md
 docs/source/nodes/prompt_maker/prompt_maker.md
@@ -419,7 +432,9 @@ tests/autorag/nodes/passagereranker/test_flag_embedding.py
 tests/autorag/nodes/passagereranker/test_flag_embedding_llm.py
 tests/autorag/nodes/passagereranker/test_jina_reranker.py
 tests/autorag/nodes/passagereranker/test_koreranker.py
+tests/autorag/nodes/passagereranker/test_mixedbreadai_reranker.py
 tests/autorag/nodes/passagereranker/test_monot5.py
+tests/autorag/nodes/passagereranker/test_openvino_reranker.py
 tests/autorag/nodes/passagereranker/test_pass_reranker.py
 tests/autorag/nodes/passagereranker/test_passage_reranker_base.py
 tests/autorag/nodes/passagereranker/test_passage_reranker_run.py
@@ -428,6 +443,7 @@ tests/autorag/nodes/passagereranker/test_sentence_transformer.py
 tests/autorag/nodes/passagereranker/test_tart.py
 tests/autorag/nodes/passagereranker/test_time_reranker.py
 tests/autorag/nodes/passagereranker/test_upr.py
+tests/autorag/nodes/passagereranker/test_voyageai_reranker.py
 tests/autorag/nodes/promptmaker/test_fstring.py
 tests/autorag/nodes/promptmaker/test_long_context_reorder.py
 tests/autorag/nodes/promptmaker/test_prompt_maker_base.py
@@ -462,6 +478,7 @@ tests/resources/simple.yaml
 tests/resources/simple_chunk.yaml
 tests/resources/simple_mock.yaml
 tests/resources/simple_parse.yaml
+tests/resources/simple_with_llm.yaml
 tests/resources/test_bm25_retrieval.pkl
 tests/resources/chunk_data/sample_parsed.parquet
 tests/resources/data_creation/raw_dir/sample1.txt

{autorag-0.3.3 → autorag-0.3.4}/AutoRAG.egg-info/requires.txt

@@ -14,7 +14,6 @@ rouge_score
 rich
 chromadb>=0.5.0
 click
-Flask
 torch
 sentencepiece
 cohere>=5.8.0
@@ -25,6 +24,10 @@ sentence-transformers
 FlagEmbedding
 llmlingua
 peft
+optimum[nncf,openvino]
+voyageai
+mixedbread-ai
+quart
 llama-index>=0.11.0
 llama-index-core>=0.11.0
 llama-index-readers-file
@@ -51,11 +54,16 @@ ipywidgets_bokeh
 AutoRAG[ko]
 AutoRAG[dev]
 AutoRAG[parse]
+AutoRAG[ja]
 
 [dev]
 ruff
 pre-commit
 
+[ja]
+sudachipy>=0.6.8
+sudachidict_core
+
 [ko]
 kiwipiepy>=0.18.0
 konlpy

autorag-0.3.4/Dockerfile

@@ -0,0 +1,61 @@
+# Base stage: Install common dependencies
+FROM python:3.10-slim AS base
+
+# Install system dependencies
+RUN apt-get update && \
+    apt-get install -y \
+    build-essential \
+    gcc \
+    libssl-dev \
+    poppler-utils \
+    tesseract-ocr \
+    tesseract-ocr-eng \
+    tesseract-ocr-kor && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set work directory
+WORKDIR /usr/src/app
+
+# Install Python dependencies
+RUN pip install --upgrade pip setuptools setuptools-scm
+COPY requirements.txt /usr/src/app/requirements.txt
+RUN pip install -r requirements.txt
+
+# Copy project files
+COPY . /usr/src/app
+RUN pip install -e ./
+
+# Test stage
+FROM base AS test
+RUN pip install pytest pytest-xdist
+
+# Ko stage
+FROM base AS ko
+RUN pip install --no-cache-dir kiwipiepy>=0.18.0 konlpy
+ENTRYPOINT ["python", "-m", "autorag.cli"]
+
+# Dev stage
+FROM base AS dev
+RUN pip install --no-cache-dir ruff pre-commit
+ENTRYPOINT ["python", "-m", "autorag.cli"]
+
+# Parsing stage
+FROM base AS parsing
+RUN pip install --no-cache-dir PyMuPDF pdfminer.six pdfplumber unstructured jq "unstructured[pdf]" "PyPDF2<3.0" pdf2image
+ENTRYPOINT ["python", "-m", "autorag.cli"]
+
+# Parsing stage
+FROM base AS all
+# TODO
+ENTRYPOINT ["python", "-m", "autorag.cli"]
+
+# Production stage (includes all features)
+FROM base AS production
+RUN pip install --no-cache-dir \
+    kiwipiepy>=0.18.0 konlpy \
+    ruff pre-commit \
+    PyMuPDF pdfminer.six pdfplumber unstructured jq "unstructured[pdf]" "PyPDF2<3.0" pdf2image
+
+COPY projects /usr/src/app/projects
+
+ENTRYPOINT ["python", "-m", "autorag.cli"]

{autorag-0.3.3 → autorag-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.3.3
+Version: 0.3.4
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -235,7 +235,6 @@ Requires-Dist: rouge_score
 Requires-Dist: rich
 Requires-Dist: chromadb>=0.5.0
 Requires-Dist: click
-Requires-Dist: Flask
 Requires-Dist: torch
 Requires-Dist: sentencepiece
 Requires-Dist: cohere>=5.8.0
@@ -246,6 +245,10 @@ Requires-Dist: sentence-transformers
 Requires-Dist: FlagEmbedding
 Requires-Dist: llmlingua
 Requires-Dist: peft
+Requires-Dist: optimum[nncf,openvino]
+Requires-Dist: voyageai
+Requires-Dist: mixedbread-ai
+Requires-Dist: quart
 Requires-Dist: llama-index>=0.11.0
 Requires-Dist: llama-index-core>=0.11.0
 Requires-Dist: llama-index-readers-file
@@ -282,10 +285,14 @@ Requires-Dist: jq; extra == "parse"
 Requires-Dist: unstructured[pdf]; extra == "parse"
 Requires-Dist: PyPDF2<3.0; extra == "parse"
 Requires-Dist: pdf2image; extra == "parse"
+Provides-Extra: ja
+Requires-Dist: sudachipy>=0.6.8; extra == "ja"
+Requires-Dist: sudachidict_core; extra == "ja"
 Provides-Extra: all
 Requires-Dist: AutoRAG[ko]; extra == "all"
 Requires-Dist: AutoRAG[dev]; extra == "all"
 Requires-Dist: AutoRAG[parse]; extra == "all"
+Requires-Dist: AutoRAG[ja]; extra == "all"
 
 # AutoRAG
 
@@ -655,14 +662,17 @@ You can run this pipeline as an API server.
 Check out the API endpoint at [here](deploy/api_endpoint.md).
 
 ```python
-from autorag.deploy import Runner
+import nest_asyncio
+from autorag.deploy import ApiRunner
 
-runner = Runner.from_trial_folder('/your/path/to/trial_dir')
+nest_asyncio.apply()
+
+runner = ApiRunner.from_trial_folder('/your/path/to/trial_dir')
 runner.run_api_server()
 ```
 
 ```bash
-autorag run_api --config_path your/path/to/pipeline.yaml --host 0.0.0.0 --port 8000
+autorag run_api --trial_dir your/path/to/trial_dir --host 0.0.0.0 --port 8000
 ```
 
 The cli command uses extracted config YAML file. If you want to know it more, check out [here](https://docs.auto-rag.com/tutorial.html#extract-pipeline-and-evaluate-test-dataset).

{autorag-0.3.3 → autorag-0.3.4}/README.md

@@ -366,14 +366,17 @@ You can run this pipeline as an API server.
 Check out the API endpoint at [here](deploy/api_endpoint.md).
 
 ```python
-from autorag.deploy import Runner
+import nest_asyncio
+from autorag.deploy import ApiRunner
 
-runner = Runner.from_trial_folder('/your/path/to/trial_dir')
+nest_asyncio.apply()
+
+runner = ApiRunner.from_trial_folder('/your/path/to/trial_dir')
 runner.run_api_server()
 ```
 
 ```bash
-autorag run_api --config_path your/path/to/pipeline.yaml --host 0.0.0.0 --port 8000
+autorag run_api --trial_dir your/path/to/trial_dir --host 0.0.0.0 --port 8000
 ```
 
 The cli command uses extracted config YAML file. If you want to know it more, check out [here](https://docs.auto-rag.com/tutorial.html#extract-pipeline-and-evaluate-test-dataset).

autorag-0.3.4/autorag/VERSION

@@ -0,0 +1 @@
+0.3.4

{autorag-0.3.3 → autorag-0.3.4}/autorag/cli.py

@@ -7,10 +7,11 @@ from pathlib import Path
 from typing import Optional
 
 import click
+import nest_asyncio
 
 from autorag import dashboard
-from autorag.deploy import Runner
 from autorag.deploy import extract_best_config as original_extract_best_config
+from autorag.deploy.api import ApiRunner
 from autorag.evaluator import Evaluator
 from autorag.validator import Validator
 
@@ -48,15 +49,27 @@ def evaluate(config, qa_data_path, corpus_data_path, project_dir):
 
 
 @click.command()
-@click.option("--config_path", type=str, help="Path to extracted config yaml file.")
+@click.option(
+	"--config_path", type=str, help="Path to extracted config yaml file.", default=None
+)
 @click.option("--host", type=str, default="0.0.0.0", help="Host address")
 @click.option("--port", type=int, default=8000, help="Port number")
+@click.option(
+	"--trial_dir",
+	type=click.Path(file_okay=False, dir_okay=True, exists=True),
+	default=None,
+	help="Path to trial directory.",
+)
 @click.option(
 	"--project_dir", help="Path to project directory.", type=str, default=None
 )
-def run_api(config_path, host, port, project_dir):
-	runner = Runner.from_yaml(config_path, project_dir=project_dir)
+def run_api(config_path, host, port, trial_dir, project_dir):
+	if trial_dir is None:
+		runner = ApiRunner.from_yaml(config_path, project_dir=project_dir)
+	else:
+		runner = ApiRunner.from_trial_folder(trial_dir)
 	logger.info(f"Running API server at {host}:{port}...")
+	nest_asyncio.apply()
 	runner.run_api_server(host, port)
 
 

{autorag-0.3.3 → autorag-0.3.4}/autorag/data/chunk/base.py

@@ -115,7 +115,14 @@ def add_file_name(
 				zip(chunk_texts, file_names),
 			)
 		)
+	elif file_name_language == 'ja':
+				return list(
+			map(
+				lambda x: f"ファイル名: {x[1]}\n 内容: {x[0]}",
+				zip(chunk_texts, file_names),
+			)
+		)
 	else:
 		raise ValueError(
-			f"Unsupported file_name_language: {file_name_language}. Choose from 'en' or 'ko'."
+			f"Unsupported file_name_language: {file_name_language}. Choose from 'en' ,'ko' or 'ja."
 		)

{autorag-0.3.3 → autorag-0.3.4}/autorag/data/legacy/qacreation/base.py

@@ -166,7 +166,7 @@ def make_qa_with_existing_qa(
 		content_size = len(corpus_df)
 
 	logger.info("Loading local embedding model...")
-	embeddings = autorag.embedding_models[embedding_model]
+	embeddings = autorag.embedding_models[embedding_model]()
 
 	# Vector DB creation
 	if collection is None:

{autorag-0.3.3 → autorag-0.3.4}/autorag/data/qa/evolve/prompt.py

@@ -71,6 +71,40 @@ Output: """,
 				content="백신은 신체의 면역 체계를 어떻게 활용해서 질병을 예방합니까?",
 			),
 		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""提供された質問に条件に関する内容を追加して、複雑さを高めます。
+質問のContextに影響を与えるシナリオや条件を含めて、質問をより複雑にすることが目標です。
+質問を再作成するときは、次のルールに従います。
+    1. 再作成された質問は100文字を超えてはいけません。 可能であれば略語を使ってください
+    2. 再作成された質問は合理的でなければならず、人が理解して回答できるものでなければなりません。
+    3. 再作成された質問は、現在のContextで完全に答えられる必要があります。
+    4. 「提供された文」、「段落によると?」、「Contextによると」などのフレーズは質問に表示されません。
+    5. 日本語で質問を書きましょう。
+""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 植物の根の機能は何ですか？
+Context: 植物の根は土壌から水や栄養分を吸収し、植物を地面に固定し、栄養分を蓄えます。
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="植物の根は土壌栄養分と安定性に対してどのような役割をしますか？",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: ワクチンは病気をどのように予防しますか?
+Context: ワクチンは、体の免疫反応を刺激して病原体を認識し、戦う抗体を生成することで病気から守ります。
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="ワクチンは体の免疫システムをどのように活用して病気を予防しますか？",
+			),
+		],
 	},
 	"reasoning_evolve_ragas": {
 		"en": [
@@ -138,6 +172,39 @@ Output: """,
 				content="리스트가 변수들을 모아 놓은 것을 나타낸다면, 어떤 메서드를 사용해야 항목을 하나 더 추가할 수 있습니까?",
 			),
 		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""与えられたContextに基づいて既存の質問を複雑にして、様々な論理的思考が必要な質問として書き直しましょう。
+質問に答えるためには、与えられたContextの情報を使って様々な論理的思考や推論をしなければなりません。
+質問を再作成するときに従うべきルール:
+1. 再作成された質問は、Contextにある情報だけで完全に答えられる必要があります。
+2. 100文字を超える質問を作成してはいけません。 可能であれば略語を使ってください。
+3. 質問が明確で曖昧にならないようにしましょう。
+4. 「提供されたContextに基づいて」、「当該段落によると」などのフレーズは、質問に含まれてはいけません。
+5. 日本語で質問を書きましょう。""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: フランスの首都はどこですか？,
+Context: フランスは西ヨーロッパにある国です。 パリ、リヨン、マルセイユを含むいくつかの都市があります。 パリはエッフェル塔やルーブル博物館のような文化的ランドマークとして有名なだけでなく、行政の中心地としても知られています。
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="エッフェル塔と行政の中心地、二つの単語はどんな都市を指していますか？",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: Pythonでappend() メソッドは何をしますか？
+Context: Pythonで、リストは 1 つの変数に複数の項目を保存するために使用されます。 リストは、データを保存するために使用される 4 つの組み込みデータ タイプの 1 つです。 append()メソッドは、リストの最後に新しい項目を追加します。
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="リストが変数を集めたものである場合、どのメソッドを使えば項目を一つ追加することができますか？",
+			),
+		],
 	},
 	"compress_ragas": {
 		"en": [
@@ -191,5 +258,31 @@ Output: """,
 				content="초콜릿 케이크에 필요한 것은 무엇인가요?",
 			),
 		],
+		"ja": [
+			ChatMessage(
+				role=MessageRole.SYSTEM,
+				content="""与えられた質問をより間接的かつ短く書き換えます。
+目標は、質問を元の質問の本質を保ちながら、あまりストレートにならないようにすることです。
+略語などを使用して、質問をより短くします。""",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: 地球と月の間の距離はどれくらいですか？
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="月は地球からどれくらい離れていますか？",
+			),
+			ChatMessage(
+				role=MessageRole.USER,
+				content="""Question: チョコレートケーキを焼くために必要な材料は何ですか？
+Output: """,
+			),
+			ChatMessage(
+				role=MessageRole.ASSISTANT,
+				content="チョコレートケーキに必要なものは何ですか？",
+			),
+		],
 	},
 }

{autorag-0.3.3 → autorag-0.3.4}/autorag/data/qa/filter/dontknow.py

@@ -28,6 +28,16 @@ dont_know_phrases = {
 		"모르겠네유",
 		"모르겠네요",
 	],
+	"ja": [
+		"知りません",
+		"わかりません",
+		"分かりません",
+		"知らないです",
+		"よく分かってません",
+		"わかりかねます",
+		"存じません",
+		"お答えいたしかねます"
+	]
 }
 
 
@@ -60,7 +70,7 @@ async def dontknow_filter_openai(
 	:param client: The OpenAI client.
 	:param model_name: The model name.
 		You have to use gpt-4o-2024-08-06 or gpt-4o-mini-2024-07-18.
-	:param lang: The supported language is en or ko.
+	:param lang: The supported language is en, ko or ja.
 	:return: False if the row generation_gt is a "don't know" meaning.
 	"""
 	assert "generation_gt" in row.keys(), "generation_gt column is not in the row."
@@ -91,7 +101,7 @@ async def dontknow_filter_llama_index(
 	:param row: The row dict from QA dataset.
 	:param llm: The Llama index llm instance.
 		It will be good if you set max tokens to low for saving tokens.
-	:param lang: The supported language is en or ko.
+	:param lang: The supported language is en, ko or ja.
 	:return: False if the row generation_gt is a "don't know" meaning.
 	"""
 	assert "generation_gt" in row.keys(), "generation_gt column is not in the row."

{autorag-0.3.3 → autorag-0.3.4}/autorag/data/qa/filter/passage_dependency.py

@@ -31,7 +31,7 @@ async def passage_dependency_filter_openai(
 	    :param client: The OpenAI client.
 	    :param model_name: The model name.
 	            You have to use gpt-4o-2024-08-06 or gpt-4o-mini-2024-07-18.
-	    :param lang: The supported language is en or ko.
+	    :param lang: The supported language is en, ko or ja.
 	:return: False if the row question is a passage-dependent question (to be filtered).
 	"""
 	assert "query" in row.keys(), "query column is not in the row."
@@ -69,7 +69,7 @@ async def passage_dependency_filter_llama_index(
 	:param row: The row dict from QA dataset.
 	:param llm: The Llama index llm instance.
 	            It will be good if you set max tokens to low for saving tokens.
-	    :param lang: The supported language is en or ko.
+	    :param lang: The supported language is en, ko or ja.
 	:return: False if the row question is a passage-dependent question (to be filtered).
 	"""
 	assert "query" in row.keys(), "query column is not in the row."