AutoRAG 0.3.6__tar.gz → 0.3.8__tar.gz

{autorag-0.3.6 → autorag-0.3.8}/.github/workflows/docker-push.yml

@@ -67,20 +67,20 @@ jobs:
       uses: docker/build-push-action@v6.9.0
       with:
         context: .
-        file: ./docker/base/Dockerfile
+        file: Dockerfile.base
         push: true
         # push: ${{ github.event_name != 'pull_request' && matrix.variant != 'test' }}
-        tags: ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
+        tags: ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
           TARGET_STAGE=${{ matrix.variant }}
 
     - name: Tag and push 'all' for production
       run: |
-        docker pull ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
-        docker tag ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}} ${{ env.DOCKER_REPO }}:latest
-        docker push ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
-        docker push ${{ env.DOCKER_REPO }}:latest
+        docker pull ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }}
+        docker tag ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }} ${{ env.DOCKER_REPO }}:${{matrix.variant}}-latest
+        docker push ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }}
+        docker push ${{ env.DOCKER_REPO }}:${{matrix.variant}}-latest
 
     # - name: Update Docker Hub description
     #   uses: peter-evans/dockerhub-description@v3

{autorag-0.3.6 → autorag-0.3.8}/.github/workflows/gpu-docker-push.yml

@@ -67,20 +67,20 @@ jobs:
       uses: docker/build-push-action@v6.9.0
       with:
         context: .
-        file: ./docker/gpu/Dockerfile
+        file: ./Dockerfile.gpu
         push: true
         # push: ${{ github.event_name != 'pull_request' && matrix.variant != 'test' }}
-        tags: ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
+        tags: ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
           TARGET_STAGE=${{ matrix.variant }}
 
     - name: Tag and push 'all' for production
       run: |
-        docker pull ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
-        docker tag ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}} ${{ env.DOCKER_REPO }}:latest
-        docker push ${{ env.DOCKER_REPO }}:${{ env.VERSION }}-${{matrix.variant}}
-        docker push ${{ env.DOCKER_REPO }}:latest
+        docker pull ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }}
+        docker tag ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }} ${{ env.DOCKER_REPO }}:${{matrix.variant}}-latest
+        docker push ${{ env.DOCKER_REPO }}:${{matrix.variant}}-${{ env.VERSION }}
+        docker push ${{ env.DOCKER_REPO }}:${{matrix.variant}}-latest
 
     # - name: Update Docker Hub description
     #   uses: peter-evans/dockerhub-description@v3

{autorag-0.3.6 → autorag-0.3.8}/AutoRAG.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.3.6
+Version: 0.3.8
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -232,7 +232,6 @@ Requires-Dist: sacrebleu
 Requires-Dist: evaluate
 Requires-Dist: rouge_score
 Requires-Dist: rich
-Requires-Dist: chromadb>=0.5.0
 Requires-Dist: click
 Requires-Dist: cohere>=5.8.0
 Requires-Dist: tokenlog>=0.0.2
@@ -240,7 +239,11 @@ Requires-Dist: aiohttp
 Requires-Dist: voyageai
 Requires-Dist: mixedbread-ai
 Requires-Dist: llama-index-llms-bedrock
+Requires-Dist: scikit-learn
+Requires-Dist: pymilvus
+Requires-Dist: chromadb>=0.5.0
 Requires-Dist: quart
+Requires-Dist: pyngrok
 Requires-Dist: llama-index>=0.11.0
 Requires-Dist: llama-index-core>=0.11.0
 Requires-Dist: llama-index-readers-file
@@ -304,6 +307,14 @@ RAG AutoML tool for automatically finding an optimal RAG pipeline for your data.
 
 ![Thumbnail](https://github.com/user-attachments/assets/6bab243d-a4b3-431a-8ac0-fe17336ab4de)
 
+![Discord](https://img.shields.io/discord/1204010535272587264) ![PyPI - Downloads](https://img.shields.io/pypi/dm/AutoRAG)
+[![LinkedIn](https://img.shields.io/badge/LinkedIn-Connect-blue?style=flat-square&logo=linkedin)](https://www.linkedin.com/company/104375108/admin/dashboard/)
+![X (formerly Twitter) Follow](https://img.shields.io/twitter/follow/AutoRAG_HQ)
+[![Hugging Face](https://img.shields.io/badge/Hugging%20Face-Follow-orange?style=flat-square&logo=huggingface)](https://huggingface.co/AutoRAG)
+[![Static Badge](https://img.shields.io/badge/Roadmap-5D3FD3)](https://github.com/orgs/Auto-RAG/projects/1/views/2)
+
+
+
 There are many RAG pipelines and modules out there,
 but you don’t know what pipeline is great for “your own data” and "your own use-case."
 Making and evaluating all RAG modules is very time-consuming and hard to do.
@@ -322,6 +333,16 @@ Plus, join our 📞 [Discord](https://discord.gg/P4DYXfmSAs) Community.
 
 ---
 
+Do you have any difficulties in optimizing your RAG pipeline?
+Or is it hard to set up things to use AutoRAG?
+Try [**AutoRAG Cloud**](https://tally.so/r/n0jOrZ) beta.
+We will help you to run AutoRAG and optimize.
+Plus, we can help you to build RAG evaluation dataset.
+
+Starts with 9.99$ per optimization.
+
+---
+
 ## YouTube Tutorial
 
 https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/c0d23896-40c0-479f-a17b-aa2ec3183a26
@@ -608,7 +629,7 @@ node_lines:
       top_k: 3
       modules:
         - module_type: vectordb
-          embedding_model: openai
+          vectordb: default
         - module_type: bm25
         - module_type: hybrid_rrf
           weight_range: (4,80)
@@ -691,7 +712,7 @@ runner.run('your question')
 
 You can run this pipeline as an API server.
 
-Check out the API endpoint at [here](deploy/api_endpoint.md).
+Check out the API endpoint at [here](./docs/source/deploy/api_endpoint.md).
 
 ```python
 import nest_asyncio
@@ -723,6 +744,15 @@ autorag run_web --trial_path your/path/to/trial_path
 
 <img width="1491" alt="web_interface" src="https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/f6b00353-f6bb-4d8f-8740-1c264c0acbb8">
 
+### Use advanced web interface
+
+You can deploy the advanced web interface featured by [Kotaemon](https://github.com/Cinnamon/kotaemon) to the fly.io.
+Go [here](https://github.com/vkehfdl1/AutoRAG-web-kotaemon) to use it and deploy to the fly.io.
+
+Example :
+
+![Kotaemon Example](https://velog.velcdn.com/images/autorag/post/5e71b8d9-3e59-4e63-9191-355a1a5aa3a0/image.png)
+
 ## 📌 Supporting Data Creation Modules
 ![Data Creation](https://github.com/user-attachments/assets/0e5872de-2892-46b4-9ecd-e395671e324c)
 
@@ -753,7 +783,7 @@ at [here](https://edai.notion.site/Supporting-metrics-867d71caefd7401c9264dd91ba
 
 ## ☎️ FaQ
 
-🛣️ [Support plans & Roadmap](https://edai.notion.site/Support-plans-Roadmap-02ca7c97376340c393885855e2d99630?pvs=4)
+🛣️ [Roadmap](https://github.com/orgs/Auto-RAG/projects/1/views/2)
 
 💻 [Hardware Specs](https://edai.notion.site/Hardware-specs-28cefcf2a26246ffadc91e2f3dc3d61c?pvs=4)
 

{autorag-0.3.6 → autorag-0.3.8}/AutoRAG.egg-info/SOURCES.txt

@@ -3,6 +3,8 @@
 .pre-commit-config.yaml
 CODE_OF_CONDUCT.md
 CONTRIBUTING.md
+Dockerfile.base
+Dockerfile.gpu
 LICENSE
 README.md
 build_and_push.sh
@@ -182,8 +184,10 @@ autorag/schema/node.py
 autorag/utils/__init__.py
 autorag/utils/preprocess.py
 autorag/utils/util.py
-docker/base/Dockerfile
-docker/gpu/Dockerfile
+autorag/vectordb/__init__.py
+autorag/vectordb/base.py
+autorag/vectordb/chroma.py
+autorag/vectordb/milvus.py
 docs/Makefile
 docs/make.bat
 docs/requirements.txt
@@ -197,6 +201,7 @@ docs/source/structure.md
 docs/source/troubleshooting.md
 docs/source/tutorial.md
 docs/source/_static/data_creation.png
+docs/source/_static/data_creation_pipeline.png
 docs/source/_static/data_folder.png
 docs/source/_static/dcg.png
 docs/source/_static/f1_score.png
@@ -256,6 +261,7 @@ docs/source/api_spec/autorag.nodes.rst
 docs/source/api_spec/autorag.rst
 docs/source/api_spec/autorag.schema.rst
 docs/source/api_spec/autorag.utils.rst
+docs/source/api_spec/autorag.vectordb.rst
 docs/source/api_spec/modules.rst
 docs/source/data_creation/data_creation.md
 docs/source/data_creation/data_format.md
@@ -333,6 +339,9 @@ docs/source/optimization/folder_structure.md
 docs/source/optimization/optimization.md
 docs/source/optimization/strategies.md
 docs/source/roadmap/modular_rag.md
+docs/source/vectordb/chroma.md
+docs/source/vectordb/milvus.md
+docs/source/vectordb/vectordb.md
 projects/tutorial_1/config.yaml
 sample_config/chunk/chunk_full.yaml
 sample_config/chunk/chunk_ko.yaml
@@ -340,6 +349,7 @@ sample_config/chunk/simple_chunk.yaml
 sample_config/parse/parse_full.yaml
 sample_config/parse/parse_hybird.yaml
 sample_config/parse/parse_ko.yaml
+sample_config/parse/parse_multimodal.yaml
 sample_config/parse/parse_ocr.yaml
 sample_config/parse/simple_parse.yaml
 sample_config/rag/extracted_sample.yaml
@@ -373,6 +383,7 @@ sample_dataset/eli5/load_eli5_dataset.py
 sample_dataset/hotpotqa/load_hotpotqa_dataset.py
 sample_dataset/msmarco/load_msmarco_dataset.py
 sample_dataset/triviaqa/load_triviaqa_dataset.py
+tests/__init__.py
 tests/conftest.py
 tests/delete_tests.py
 tests/mock.py
@@ -491,6 +502,9 @@ tests/autorag/schema/test_module_schema.py
 tests/autorag/schema/test_node_schema.py
 tests/autorag/utils/test_preprocess.py
 tests/autorag/utils/test_util.py
+tests/autorag/vectordb/test_base.py
+tests/autorag/vectordb/test_chroma.py
+tests/autorag/vectordb/test_milvus.py
 tests/resources/README.md
 tests/resources/corpus_data_sample.parquet
 tests/resources/full.yaml
@@ -499,6 +513,7 @@ tests/resources/qa_test_data_sample.parquet
 tests/resources/sample_contents_nqa.csv
 tests/resources/simple.yaml
 tests/resources/simple_chunk.yaml
+tests/resources/simple_milvus.yaml
 tests/resources/simple_mock.yaml
 tests/resources/simple_parse.yaml
 tests/resources/simple_with_llm.yaml
@@ -601,6 +616,7 @@ tests/resources/result_project/3/config.yaml
 tests/resources/result_project/data/corpus.parquet
 tests/resources/result_project/data/qa.parquet
 tests/resources/result_project/resources/bm25_porter_stemmer.pkl
+tests/resources/result_project/resources/vectordb.yaml
 tests/resources/result_project/resources/chroma/chroma.sqlite3
 tests/resources/result_project/resources/chroma/6595d304-5270-4d9f-a267-c191366804ce/data_level0.bin
 tests/resources/result_project/resources/chroma/6595d304-5270-4d9f-a267-c191366804ce/header.bin
@@ -609,4 +625,5 @@ tests/resources/result_project/resources/chroma/6595d304-5270-4d9f-a267-c1913668
 tests/resources/sample_project/data/corpus.parquet
 tests/resources/sample_project/data/qa.parquet
 tests/resources/sample_project/resources/bm25_gpt2.pkl
-tests/resources/sample_project/resources/bm25_porter_stemmer.pkl
+tests/resources/sample_project/resources/bm25_porter_stemmer.pkl
+tests/resources/sample_project/resources/vectordb.yaml

{autorag-0.3.6 → autorag-0.3.8}/AutoRAG.egg-info/requires.txt

@@ -11,7 +11,6 @@ sacrebleu
 evaluate
 rouge_score
 rich
-chromadb>=0.5.0
 click
 cohere>=5.8.0
 tokenlog>=0.0.2
@@ -19,7 +18,11 @@ aiohttp
 voyageai
 mixedbread-ai
 llama-index-llms-bedrock
+scikit-learn
+pymilvus
+chromadb>=0.5.0
 quart
+pyngrok
 llama-index>=0.11.0
 llama-index-core>=0.11.0
 llama-index-readers-file

autorag-0.3.8/Dockerfile.base

@@ -0,0 +1,71 @@
+# Base stage: Install common dependencies
+FROM python:3.10-slim AS base
+
+# Set working directory and environment variables
+WORKDIR /usr/src/app
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1
+
+# Copy only requirements files first to leverage Docker cache
+COPY pyproject.toml ./
+
+# Install system and Python dependencies in a single layer
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    build-essential \
+    gcc \
+    libssl-dev && \
+    pip install --upgrade pip setuptools setuptools-scm && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /root/.cache/pip/*
+
+# Copy project files
+COPY . .
+
+# Install base project
+RUN pip install -e ./ && \
+    rm -rf /root/.cache/pip/*
+
+# Ko stage with minimal dependencies
+FROM base AS ko
+RUN pip install --no-cache-dir "AutoRAG[ko]" && \
+    rm -rf /root/.cache/pip/*
+ENTRYPOINT ["python", "-m", "autorag.cli"]
+
+# Dev stage with all development tools
+FROM base AS dev
+RUN pip install --no-cache-dir \
+    -r ./tests/requirements.txt \
+    -r ./docs/requirements.txt \
+    "AutoRAG[dev]" && \
+    rm -rf /root/.cache/pip/*
+ENTRYPOINT ["python", "-m", "autorag.cli"]
+
+# Ja stage with Japanese support
+FROM base AS ja
+RUN pip install --no-cache-dir "AutoRAG[ja]" && \
+    rm -rf /root/.cache/pip/*
+ENTRYPOINT ["python", "-m", "autorag.cli"]
+
+# API stage with minimal footprint
+FROM base AS api
+RUN rm -rf \
+    ./sample_dataset \
+    ./tests \
+    ./docs \
+    /root/.cache/pip/*
+ENTRYPOINT ["python", "-m", "autorag.cli"]
+
+# Parsing stage with OCR support
+FROM base AS parsing
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    poppler-utils \
+    tesseract-ocr \
+    tesseract-ocr-eng \
+    tesseract-ocr-kor && \
+    pip install --no-cache-dir "AutoRAG[parse]" && \
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /root/.cache/pip/*
+ENTRYPOINT ["python", "-m", "autorag.cli"]

{autorag-0.3.6 → autorag-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AutoRAG
-Version: 0.3.6
+Version: 0.3.8
 Summary: Automatically Evaluate RAG pipelines with your own data. Find optimal structure for new RAG product.
 Author-email: Marker-Inc <vkehfdl1@gmail.com>
 License:                                  Apache License
@@ -232,7 +232,6 @@ Requires-Dist: sacrebleu
 Requires-Dist: evaluate
 Requires-Dist: rouge_score
 Requires-Dist: rich
-Requires-Dist: chromadb>=0.5.0
 Requires-Dist: click
 Requires-Dist: cohere>=5.8.0
 Requires-Dist: tokenlog>=0.0.2
@@ -240,7 +239,11 @@ Requires-Dist: aiohttp
 Requires-Dist: voyageai
 Requires-Dist: mixedbread-ai
 Requires-Dist: llama-index-llms-bedrock
+Requires-Dist: scikit-learn
+Requires-Dist: pymilvus
+Requires-Dist: chromadb>=0.5.0
 Requires-Dist: quart
+Requires-Dist: pyngrok
 Requires-Dist: llama-index>=0.11.0
 Requires-Dist: llama-index-core>=0.11.0
 Requires-Dist: llama-index-readers-file
@@ -304,6 +307,14 @@ RAG AutoML tool for automatically finding an optimal RAG pipeline for your data.
 
 ![Thumbnail](https://github.com/user-attachments/assets/6bab243d-a4b3-431a-8ac0-fe17336ab4de)
 
+![Discord](https://img.shields.io/discord/1204010535272587264) ![PyPI - Downloads](https://img.shields.io/pypi/dm/AutoRAG)
+[![LinkedIn](https://img.shields.io/badge/LinkedIn-Connect-blue?style=flat-square&logo=linkedin)](https://www.linkedin.com/company/104375108/admin/dashboard/)
+![X (formerly Twitter) Follow](https://img.shields.io/twitter/follow/AutoRAG_HQ)
+[![Hugging Face](https://img.shields.io/badge/Hugging%20Face-Follow-orange?style=flat-square&logo=huggingface)](https://huggingface.co/AutoRAG)
+[![Static Badge](https://img.shields.io/badge/Roadmap-5D3FD3)](https://github.com/orgs/Auto-RAG/projects/1/views/2)
+
+
+
 There are many RAG pipelines and modules out there,
 but you don’t know what pipeline is great for “your own data” and "your own use-case."
 Making and evaluating all RAG modules is very time-consuming and hard to do.
@@ -322,6 +333,16 @@ Plus, join our 📞 [Discord](https://discord.gg/P4DYXfmSAs) Community.
 
 ---
 
+Do you have any difficulties in optimizing your RAG pipeline?
+Or is it hard to set up things to use AutoRAG?
+Try [**AutoRAG Cloud**](https://tally.so/r/n0jOrZ) beta.
+We will help you to run AutoRAG and optimize.
+Plus, we can help you to build RAG evaluation dataset.
+
+Starts with 9.99$ per optimization.
+
+---
+
 ## YouTube Tutorial
 
 https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/c0d23896-40c0-479f-a17b-aa2ec3183a26
@@ -608,7 +629,7 @@ node_lines:
       top_k: 3
       modules:
         - module_type: vectordb
-          embedding_model: openai
+          vectordb: default
         - module_type: bm25
         - module_type: hybrid_rrf
           weight_range: (4,80)
@@ -691,7 +712,7 @@ runner.run('your question')
 
 You can run this pipeline as an API server.
 
-Check out the API endpoint at [here](deploy/api_endpoint.md).
+Check out the API endpoint at [here](./docs/source/deploy/api_endpoint.md).
 
 ```python
 import nest_asyncio
@@ -723,6 +744,15 @@ autorag run_web --trial_path your/path/to/trial_path
 
 <img width="1491" alt="web_interface" src="https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/f6b00353-f6bb-4d8f-8740-1c264c0acbb8">
 
+### Use advanced web interface
+
+You can deploy the advanced web interface featured by [Kotaemon](https://github.com/Cinnamon/kotaemon) to the fly.io.
+Go [here](https://github.com/vkehfdl1/AutoRAG-web-kotaemon) to use it and deploy to the fly.io.
+
+Example :
+
+![Kotaemon Example](https://velog.velcdn.com/images/autorag/post/5e71b8d9-3e59-4e63-9191-355a1a5aa3a0/image.png)
+
 ## 📌 Supporting Data Creation Modules
 ![Data Creation](https://github.com/user-attachments/assets/0e5872de-2892-46b4-9ecd-e395671e324c)
 
@@ -753,7 +783,7 @@ at [here](https://edai.notion.site/Supporting-metrics-867d71caefd7401c9264dd91ba
 
 ## ☎️ FaQ
 
-🛣️ [Support plans & Roadmap](https://edai.notion.site/Support-plans-Roadmap-02ca7c97376340c393885855e2d99630?pvs=4)
+🛣️ [Roadmap](https://github.com/orgs/Auto-RAG/projects/1/views/2)
 
 💻 [Hardware Specs](https://edai.notion.site/Hardware-specs-28cefcf2a26246ffadc91e2f3dc3d61c?pvs=4)
 

{autorag-0.3.6 → autorag-0.3.8}/README.md

@@ -4,6 +4,14 @@ RAG AutoML tool for automatically finding an optimal RAG pipeline for your data.
 
 ![Thumbnail](https://github.com/user-attachments/assets/6bab243d-a4b3-431a-8ac0-fe17336ab4de)
 
+![Discord](https://img.shields.io/discord/1204010535272587264) ![PyPI - Downloads](https://img.shields.io/pypi/dm/AutoRAG)
+[![LinkedIn](https://img.shields.io/badge/LinkedIn-Connect-blue?style=flat-square&logo=linkedin)](https://www.linkedin.com/company/104375108/admin/dashboard/)
+![X (formerly Twitter) Follow](https://img.shields.io/twitter/follow/AutoRAG_HQ)
+[![Hugging Face](https://img.shields.io/badge/Hugging%20Face-Follow-orange?style=flat-square&logo=huggingface)](https://huggingface.co/AutoRAG)
+[![Static Badge](https://img.shields.io/badge/Roadmap-5D3FD3)](https://github.com/orgs/Auto-RAG/projects/1/views/2)
+
+
+
 There are many RAG pipelines and modules out there,
 but you don’t know what pipeline is great for “your own data” and "your own use-case."
 Making and evaluating all RAG modules is very time-consuming and hard to do.
@@ -22,6 +30,16 @@ Plus, join our 📞 [Discord](https://discord.gg/P4DYXfmSAs) Community.
 
 ---
 
+Do you have any difficulties in optimizing your RAG pipeline?
+Or is it hard to set up things to use AutoRAG?
+Try [**AutoRAG Cloud**](https://tally.so/r/n0jOrZ) beta.
+We will help you to run AutoRAG and optimize.
+Plus, we can help you to build RAG evaluation dataset.
+
+Starts with 9.99$ per optimization.
+
+---
+
 ## YouTube Tutorial
 
 https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/c0d23896-40c0-479f-a17b-aa2ec3183a26
@@ -308,7 +326,7 @@ node_lines:
       top_k: 3
       modules:
         - module_type: vectordb
-          embedding_model: openai
+          vectordb: default
         - module_type: bm25
         - module_type: hybrid_rrf
           weight_range: (4,80)
@@ -391,7 +409,7 @@ runner.run('your question')
 
 You can run this pipeline as an API server.
 
-Check out the API endpoint at [here](deploy/api_endpoint.md).
+Check out the API endpoint at [here](./docs/source/deploy/api_endpoint.md).
 
 ```python
 import nest_asyncio
@@ -423,6 +441,15 @@ autorag run_web --trial_path your/path/to/trial_path
 
 <img width="1491" alt="web_interface" src="https://github.com/Marker-Inc-Korea/AutoRAG/assets/96727832/f6b00353-f6bb-4d8f-8740-1c264c0acbb8">
 
+### Use advanced web interface
+
+You can deploy the advanced web interface featured by [Kotaemon](https://github.com/Cinnamon/kotaemon) to the fly.io.
+Go [here](https://github.com/vkehfdl1/AutoRAG-web-kotaemon) to use it and deploy to the fly.io.
+
+Example :
+
+![Kotaemon Example](https://velog.velcdn.com/images/autorag/post/5e71b8d9-3e59-4e63-9191-355a1a5aa3a0/image.png)
+
 ## 📌 Supporting Data Creation Modules
 ![Data Creation](https://github.com/user-attachments/assets/0e5872de-2892-46b4-9ecd-e395671e324c)
 
@@ -453,7 +480,7 @@ at [here](https://edai.notion.site/Supporting-metrics-867d71caefd7401c9264dd91ba
 
 ## ☎️ FaQ
 
-🛣️ [Support plans & Roadmap](https://edai.notion.site/Support-plans-Roadmap-02ca7c97376340c393885855e2d99630?pvs=4)
+🛣️ [Roadmap](https://github.com/orgs/Auto-RAG/projects/1/views/2)
 
 💻 [Hardware Specs](https://edai.notion.site/Hardware-specs-28cefcf2a26246ffadc91e2f3dc3d61c?pvs=4)
 

autorag-0.3.8/autorag/VERSION

@@ -0,0 +1 @@
+0.3.8

{autorag-0.3.6 → autorag-0.3.8}/autorag/chunker.py

@@ -1,8 +1,6 @@
-import json
 import logging
 import os
 import shutil
-from datetime import datetime
 from typing import Optional
 
 import pandas as pd
@@ -32,13 +30,11 @@ class Chunker:
 		return cls(parsed_result, project_dir)
 
 	def start_chunking(self, yaml_path: str):
-		trial_name = self.__get_new_trial_name()
-		self.__make_trial_dir(trial_name)
+		if not os.path.exists(self.project_dir):
+			os.makedirs(self.project_dir)
 
 		# Copy YAML file to the trial directory
-		shutil.copy(
-			yaml_path, os.path.join(self.project_dir, trial_name, "chunk_config.yaml")
-		)
+		shutil.copy(yaml_path, os.path.join(self.project_dir, "chunk_config.yaml"))
 
 		# load yaml file
 		modules = load_yaml(yaml_path)
@@ -50,32 +46,6 @@ class Chunker:
 			modules=input_modules,
 			module_params=input_params,
 			parsed_result=self.parsed_raw,
-			trial_path=os.path.join(self.project_dir, trial_name),
+			project_dir=self.project_dir,
 		)
 		logger.info("Chunking Done!")
-
-	def __get_new_trial_name(self) -> str:
-		trial_json_path = os.path.join(self.project_dir, "trial.json")
-		if not os.path.exists(trial_json_path):
-			return "0"
-		with open(trial_json_path, "r") as f:
-			trial_json = json.load(f)
-		return str(int(trial_json[-1]["trial_name"]) + 1)
-
-	def __make_trial_dir(self, trial_name: str):
-		trial_json_path = os.path.join(self.project_dir, "trial.json")
-		if os.path.exists(trial_json_path):
-			with open(trial_json_path, "r") as f:
-				trial_json = json.load(f)
-		else:
-			trial_json = []
-
-		trial_json.append(
-			{
-				"trial_name": trial_name,
-				"start_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-			}
-		)
-		os.makedirs(os.path.join(self.project_dir, trial_name))
-		with open(trial_json_path, "w") as f:
-			json.dump(trial_json, f, indent=4)

{autorag-0.3.6 → autorag-0.3.8}/autorag/cli.py

@@ -47,7 +47,7 @@ def cli():
 )
 def evaluate(config, qa_data_path, corpus_data_path, project_dir, skip_validation):
 	if not config.endswith(".yaml") and not config.endswith(".yml"):
-		raise ValueError(f"Config file {config} is not a parquet file.")
+		raise ValueError(f"Config file {config} is not a yaml or yml file.")
 	if not os.path.exists(config):
 		raise ValueError(f"Config file {config} does not exist.")
 	evaluator = Evaluator(qa_data_path, corpus_data_path, project_dir=project_dir)
@@ -159,7 +159,7 @@ def extract_best_config(trial_path: str, output_path: str):
 def restart_evaluate(trial_path):
 	if not os.path.exists(trial_path):
 		raise ValueError(f"trial_path {trial_path} does not exist.")
-	project_dir = pathlib.PurePath(trial_path).parent
+	project_dir = str(pathlib.PurePath(trial_path).parent)
 	qa_data_path = os.path.join(project_dir, "data", "qa.parquet")
 	corpus_data_path = os.path.join(project_dir, "data", "corpus.parquet")
 	evaluator = Evaluator(qa_data_path, corpus_data_path, project_dir)

{autorag-0.3.6 → autorag-0.3.8}/autorag/data/chunk/base.py

@@ -115,8 +115,8 @@ def add_file_name(
 				zip(chunk_texts, file_names),
 			)
 		)
-	elif file_name_language == 'ja':
-				return list(
+	elif file_name_language == "ja":
+		return list(
 			map(
 				lambda x: f"ファイル名: {x[1]}\n 内容: {x[0]}",
 				zip(chunk_texts, file_names),

{autorag-0.3.6 → autorag-0.3.8}/autorag/data/chunk/run.py

@@ -9,7 +9,7 @@ def run_chunker(
 	modules: List[Callable],
 	module_params: List[Dict],
 	parsed_result: pd.DataFrame,
-	trial_path: str,
+	project_dir: str,
 ):
 	results, execution_times = zip(
 		*map(
@@ -21,7 +21,7 @@ def run_chunker(
 
 	# save results to parquet files
 	filepaths = list(
-		map(lambda x: os.path.join(trial_path, f"{x}.parquet"), range(len(modules)))
+		map(lambda x: os.path.join(project_dir, f"{x}.parquet"), range(len(modules)))
 	)
 	list(map(lambda x: x[0].to_parquet(x[1], index=False), zip(results, filepaths)))
 	filenames = list(map(lambda x: os.path.basename(x), filepaths))
@@ -34,5 +34,5 @@ def run_chunker(
 			"execution_time": average_times,
 		}
 	)
-	summary_df.to_csv(os.path.join(trial_path, "summary.csv"), index=False)
+	summary_df.to_csv(os.path.join(project_dir, "summary.csv"), index=False)
 	return summary_df

{autorag-0.3.6 → autorag-0.3.8}/autorag/data/legacy/qacreation/base.py

@@ -144,6 +144,7 @@ def make_qa_with_existing_qa(
 	:param kwargs: The keyword arguments for qa_creation_func.
 	:return: QA dataset dataframe.
 	"""
+	raise DeprecationWarning("This function is deprecated.")
 	assert (
 		"query" in existing_query_df.columns
 	), "existing_query_df must have 'query' column."