auto-coder 0.1.291__py3-none-any.whl → 0.1.293__py3-none-any.whl

{auto_coder-0.1.291.dist-info → auto_coder-0.1.293.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.291
+Version: 0.1.293
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -36,6 +36,7 @@ Requires-Dist: google-generativeai
 Requires-Dist: protobuf
 Requires-Dist: azure-cognitiveservices-speech
 Requires-Dist: real-agent
+Requires-Dist: duckdb
 Requires-Dist: python-docx
 Requires-Dist: docx2txt
 Requires-Dist: pdf2image

{auto_coder-0.1.291.dist-info → auto_coder-0.1.293.dist-info}/RECORD

@@ -1,18 +1,18 @@
 autocoder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/auto_coder.py,sha256=zKqfMdm2F419hrNGaosW4kYJ3IbaxBKlpTlVl6JFWmE,65563
 autocoder/auto_coder_lang.py,sha256=Rtupq6N3_HT7JRhDKdgCBcwRaiAnyCOR_Gsp4jUomrI,3229
-autocoder/auto_coder_rag.py,sha256=NGg0X1E4RltWHDDvyfwPPv2DiaBP5cRr9FGBpzL85oo,33483
+autocoder/auto_coder_rag.py,sha256=yhwRh_TJZyBxBCmUusZ8h5guU42i0Z6UJ10mT0FH3Rc,33857
 autocoder/auto_coder_rag_client_mcp.py,sha256=QRxUbjc6A8UmDMQ8lXgZkjgqtq3lgKYeatJbDY6rSo0,6270
 autocoder/auto_coder_rag_mcp.py,sha256=-RrjNwFaS2e5v8XDIrKR-zlUNUE8UBaeOtojffBrvJo,8521
 autocoder/auto_coder_runner.py,sha256=w-4MCKhOFaoABcDfVoZoonF59UyRso3kghimQYLz3NA,100851
-autocoder/auto_coder_server.py,sha256=6YQweNEKUrGAZ3yPvw8_qlNZJYLVSVUXGrn1K6udLts,20413
+autocoder/auto_coder_server.py,sha256=E3Z829TPSooRSNhuh3_x9yaZi0f5G0Lm0ntoZhjGaoQ,20576
 autocoder/benchmark.py,sha256=Ypomkdzd1T3GE6dRICY3Hj547dZ6_inqJbBJIp5QMco,4423
 autocoder/chat_auto_coder.py,sha256=z_Kqd7CAecuNMa77kJn7iko2zTdko-4-o72a58H-_s8,24655
 autocoder/chat_auto_coder_lang.py,sha256=CjsiJsUaWr-TJBCDDlDNnFpCDTd-itJhd9aid9DKlp8,20542
 autocoder/command_args.py,sha256=9aYJ-AmPxP1sQh6ciw04FWHjSn31f2W9afXFwo8wgx4,30441
 autocoder/lang.py,sha256=U6AjVV8Rs1uLyjFCZ8sT6WWuNUxMBqkXXIOs4S120uk,14511
 autocoder/models.py,sha256=AyoZ-Pzy0oyYUmWCxOIRiOImsqboSfRET7LO9-UOuxI,11172
-autocoder/version.py,sha256=xRk7_FB9HaCsWF0se7FARGQ80UT42_XGKM6sHbcSNCc,23
+autocoder/version.py,sha256=uJLvEc9fkxd409iL_wj7Xexi0uD8yIeEHv4m5yx5T6E,23
 autocoder/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/agent/auto_demand_organizer.py,sha256=NWSAEsEk94vT3lGjfo25kKLMwYdPcpy9e-i21txPasQ,6942
 autocoder/agent/auto_filegroup.py,sha256=CW7bqp0FW1GIEMnl-blyAc2UGT7O9Mom0q66ITz1ckM,6635
@@ -31,7 +31,7 @@ autocoder/commands/auto_web.py,sha256=_449f4rCoRG7Sv0SB0hIBRFLPLPJ5DgWW4DlI22a3X
 autocoder/commands/tools.py,sha256=lanjoBGR6H8HDJSY3KrM6ibrtHZbgKX6mKJHSSE66dg,20493
 autocoder/common/JupyterClient.py,sha256=O-wi6pXeAEYhAY24kDa0BINrLYvKS6rKyWe98pDClS0,2816
 autocoder/common/ShellClient.py,sha256=fM1q8t_XMSbLBl2zkCNC2J9xuyKN3eXzGm6hHhqL2WY,2286
-autocoder/common/__init__.py,sha256=zvNkIt388i0afI-Mt6ekwl406mllvxPgBEyzLYWh0vk,13265
+autocoder/common/__init__.py,sha256=LE-HHb_HwpWwRKwbwDlTqRzMV6nTzqF3RB55zVMDS3c,13656
 autocoder/common/anything2images.py,sha256=0ILBbWzY02M-CiWB-vzuomb_J1hVdxRcenAfIrAXq9M,25283
 autocoder/common/anything2img.py,sha256=iZQmg8srXlD7N5uGl5b_ONKJMBjYoW8kPmokkG6ISF0,10118
 autocoder/common/audio.py,sha256=Kn9nWKQddWnUrAz0a_ZUgjcu4VUU_IcZBigT7n3N3qc,7439
@@ -66,7 +66,7 @@ autocoder/common/image_to_page.py,sha256=yWiTJQ49Lm3j0FngiJhQ9u7qayqE_bOGb8Rk0Tm
 autocoder/common/index_import_export.py,sha256=h758AYY1df6JMTKUXYmMkSgxItfymDt82XT7O-ygEuw,4565
 autocoder/common/interpreter.py,sha256=62-dIakOunYB4yjmX8SHC0Gdy2h8NtxdgbpdqRZJ5vk,2833
 autocoder/common/llm_rerank.py,sha256=FbvtCzaR661Mt2wn0qsuiEL1Y3puD6jeIJS4zg_e7Bs,3260
-autocoder/common/mcp_hub.py,sha256=RPp7bnW6ij2EmBJMg2a5TN3U9G4oX_gH_vQKsIg7t40,14934
+autocoder/common/mcp_hub.py,sha256=ymy580rkv8kFx2zwQFpMg03s9K8KWsJP3dkfjoYbWSU,16573
 autocoder/common/mcp_server.py,sha256=gKaQDQWeRZgHtR9UnuxHVgVbo0acrT9qA1kwtgDpHZU,16551
 autocoder/common/mcp_tools.py,sha256=KsLvRrB6pvmebqd-lDaSH6IBJR0AIxWRE-dtCEG_w9k,12485
 autocoder/common/memory_manager.py,sha256=2ZjYG7BPyvbYalZBF6AM_G5e10Qkw_zrqtD4Zd7GSsQ,3663
@@ -84,6 +84,7 @@ autocoder/common/text.py,sha256=KGRQq314GHBmY4MWG8ossRoQi1_DTotvhxchpn78c-k,1003
 autocoder/common/types.py,sha256=PXTETrsTvhLE49jqAeUKGySvxBN9pjeyCgRHLDYdd9U,664
 autocoder/common/utils_code_auto_generate.py,sha256=oiBjdCgdcQErfhMozFdHxkU84WmDo2euBA86yezha-g,3597
 autocoder/common/mcp_servers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+autocoder/common/mcp_servers/mcp_server_gpt4o_mini_search.py,sha256=z-c3zq0YT7wK2XK2t-tDxdXFTUtCFDfvyGTYaYwRgtM,5661
 autocoder/common/mcp_servers/mcp_server_perplexity.py,sha256=IXTyMpd1CQcBLzVinA-_OIOHoNmbzvuW6pXIadaKHJE,5533
 autocoder/data/byzerllm.md,sha256=SGCMpEaUQ0ysPxQsgzyyp5sgvEr8dZsxEGAfVcPBIq0,47741
 autocoder/data/tokenizer.json,sha256=7Lb5_DaYlDRvBRH0B0ynXO5c1fOwbQLxujX805-OEh0,7847602
@@ -115,13 +116,13 @@ autocoder/privacy/__init__.py,sha256=LnIVvGu_K66zCE-yhN_-dPO8R80pQyedCsXJ7wRqQaI
 autocoder/privacy/model_filter.py,sha256=-N9ZvxxDKpxU7hkn-tKv-QHyXjvkCopUaKgvJwTOGQs,3369
 autocoder/pyproject/__init__.py,sha256=ms-A_pocgGv0oZPEW8JAdXi7G-VSVhkQ6CnWFe535Ec,14477
 autocoder/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-autocoder/rag/api_server.py,sha256=6daDd5tF_Z69cogl-nz-8ogvtKn-BNUdnnpXXxFK0uo,9419
+autocoder/rag/api_server.py,sha256=gsk450_B-qGtBwJ1niG9-QFJAG0RGr2s2KdiMrzzbyQ,9582
 autocoder/rag/conversation_to_queries.py,sha256=xwmErn4WbdADnhK1me-h_6fV3KYrl_y1qPNQl1aoI6o,4810
 autocoder/rag/doc_filter.py,sha256=UduVO2mlrngwJICrefjDJTYfdmQ4GcRXrfWDQ7xXksk,14206
-autocoder/rag/document_retriever.py,sha256=MGn6oIPo49BbRC99xmLMFkZrpHfcDfKoGYqWxXF554U,8051
+autocoder/rag/document_retriever.py,sha256=5BDqKVJqLPScEnua5S5suXhWuCaALIfPf5obXeJoWfs,8461
 autocoder/rag/lang.py,sha256=_jmUtxZDG1fmF4b2mhMJbYS1YQDb2ZE8nyAn5_vrvjA,3350
 autocoder/rag/llm_wrapper.py,sha256=Ht5GF5yJtrztoliujsZzx_ooWZmHkd5xLZKcGEiicZw,4303
-autocoder/rag/long_context_rag.py,sha256=THQakGbrr-kOn8Mu4PdJDMiiPq02FNZxZZUM8Du2YCw,41848
+autocoder/rag/long_context_rag.py,sha256=6rqq0pvYe9N4TvyLwd2OB21ZUrPC4FfxZuks0weAz4A,41935
 autocoder/rag/qa_conversation_strategy.py,sha256=_BFdgit2KkUkW_82jE67QLYS_d8BsGhU1pG73YhHJgE,5744
 autocoder/rag/rag_config.py,sha256=8LwFcTd8OJWWwi1_WY4IzjqgtT6RyE2j4PjxS5cCTDE,802
 autocoder/rag/rag_entry.py,sha256=6TKtErZ0Us9XSV6HgRKXA6yR3SiZGPHpynOKSaR1wgE,2463
@@ -143,6 +144,7 @@ autocoder/rag/cache/byzer_storage_cache.py,sha256=okmNUDRCDv81JOfBuTspmTxf8ltYmY
 autocoder/rag/cache/cache_result_merge.py,sha256=VnTdbT2OMBmWl_83bqds97d9_M33IhPNX8tF7KH2GMM,10556
 autocoder/rag/cache/file_monitor_cache.py,sha256=OdSXTH3vo6inAzkN5d55I0RN03GUlSlnUEKmXpjFl78,9443
 autocoder/rag/cache/local_byzer_storage_cache.py,sha256=7_6zCRY3BiCM0ec3U96i1G4l2SzmAedaTfkArNkMfQU,31925
+autocoder/rag/cache/local_duckdb_storage_cache.py,sha256=4lnxjwluKqVWWfmd4giJ81O6bZlqEhesHfHGgEFx55I,25128
 autocoder/rag/cache/rag_file_meta.py,sha256=RQ3n4wfkHlB-1ljS3sFSi8ijbsUPeIqBSgjmmbRuwRI,20521
 autocoder/rag/cache/simple_cache.py,sha256=j9dxhei-Nwq9FJrrGOWhaDIDSb_Iz6JSojT1pelS9k4,13084
 autocoder/rag/loaders/__init__.py,sha256=EQHEZ5Cmz-mGP2SllUTvcIbYCnF7W149dNpNItfs0yE,304
@@ -181,9 +183,9 @@ autocoder/utils/types.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/utils/auto_coder_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/utils/auto_coder_utils/chat_stream_out.py,sha256=lkJ_A-sYU36JMzjFWkk3pR6uos8oZHYt9GPsPe_CPAo,11766
 autocoder/utils/chat_auto_coder_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-auto_coder-0.1.291.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-auto_coder-0.1.291.dist-info/METADATA,sha256=P_4fuBpcjGDCIfmcSYtOHKqqvP2hUmphsz0BddB0r0w,2643
-auto_coder-0.1.291.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-auto_coder-0.1.291.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
-auto_coder-0.1.291.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
-auto_coder-0.1.291.dist-info/RECORD,,
+auto_coder-0.1.293.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+auto_coder-0.1.293.dist-info/METADATA,sha256=wM3jXJXkDW9JVdM0Oy1EVukWKunNk2NKrPjRn658wK4,2665
+auto_coder-0.1.293.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+auto_coder-0.1.293.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
+auto_coder-0.1.293.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
+auto_coder-0.1.293.dist-info/RECORD,,

autocoder/auto_coder_rag.py

@@ -179,6 +179,14 @@ def main(input_args: Optional[List[str]] = None):
     build_index_parser = subparsers.add_parser(
         "build_hybrid_index", help="Build hybrid index for RAG"
     )
+
+    build_index_parser.add_argument(
+        "--rag_storage_type",
+        type=str,
+        default="duckdb",
+        help="The storage type of the RAG, duckdb or byzer-storage",
+    )
+
     build_index_parser.add_argument(
         "--quick", action="store_true", help="Skip system initialization"
     )
@@ -329,6 +337,14 @@ def main(input_args: Optional[List[str]] = None):
         action="store_true",
         help="Enable hybrid index",
     )
+
+    serve_parser.add_argument(
+        "--rag_storage_type",
+        type=str,
+        default="duckdb",
+        help="The storage type of the RAG, duckdb or byzer-storage",
+    )    
+
     serve_parser.add_argument(
         "--hybrid_index_max_output_tokens",
         type=int,

autocoder/auto_coder_server.py

@@ -34,6 +34,14 @@ import sys
 import io
 from autocoder.utils.log_capture import LogCapture
 
+# If support dotenv, use it
+if os.path.exists(".env"):
+    try:
+        from dotenv import load_dotenv
+        load_dotenv()
+    except ImportError:
+        pass
+
 def convert_yaml_config_to_str(yaml_config):
     yaml_content = yaml.safe_dump(
         yaml_config,

autocoder/common/__init__.py

@@ -292,10 +292,14 @@ class AutoCoderArgs(pydantic.BaseModel):
         
     rag_url: Optional[str] = ""
     rag_token: Optional[str] = ""
-    rag_type: Optional[str] = "storage"   
-    rag_params_max_tokens: Optional[int] = 4096 
-    rag_doc_filter_relevance: Optional[int] = 5
-    rag_context_window_limit: Optional[int] = 120000 
+    rag_type: Optional[str] = "storage"
+    rag_storage_type: Optional[str] = "duckdb"  # 向量化存储类型 byzer-storage | duckdb
+    rag_params_max_tokens: Optional[int] = 500000 
+    rag_doc_filter_relevance: Optional[int] = 2
+    rag_context_window_limit: Optional[int] = 120000
+    rag_duckdb_vector_dim: Optional[int] = 1024  # DuckDB 向量化存储的维度
+    rag_duckdb_query_similarity: Optional[float] = 0.1  # DuckDB 向量化检索 相似度 阈值
+    rag_duckdb_query_top_k: Optional[int] = 10000  # DuckDB 向量化检索 返回 TopK个结果(且大于相似度)
     # rag 本地图床地址
     local_image_host: Optional[str] = ""
     rag_recall_max_queries: Optional[int] = 5

autocoder/common/mcp_hub.py

@@ -2,8 +2,12 @@ import os
 import json
 import asyncio
 import aiohttp
+import importlib
+import pkgutil
+import re
+import inspect
 from datetime import datetime, timedelta
-from typing import Dict, List, Optional, Any, Set, Optional
+from typing import Dict, List, Optional, Any, Set, Optional, Tuple
 from pathlib import Path
 from pydantic import BaseModel, Field
 
@@ -62,23 +66,53 @@ class McpConnection:
         self.session = session        
 
 
-MCP_PERPLEXITY_SERVER = '''
-{
-    "perplexity": {
-        "command": "python",
-        "args": [
-            "-m", "autocoder.common.mcp_servers.mcp_server_perplexity"
-        ],
-        "env": {
-            "PERPLEXITY_API_KEY": "{{PERPLEXITY_API_KEY}}"
-        }
-    }
-}
-'''
-
-MCP_BUILD_IN_SERVERS = {
-    "perplexity": json.loads(MCP_PERPLEXITY_SERVER)["perplexity"]
-}
+def _generate_server_configs() -> Tuple[Dict[str, Any], Dict[str, str]]:
+    """
+    Scan the autocoder.common.mcp_servers directory for mcp_server_*.py files
+    and generate server configurations.
+    
+    Returns:
+        Tuple of (built-in servers dict, JSON templates dict)
+    """
+    servers = {}
+    templates = {}
+    
+    try:
+        package_name = "autocoder.common.mcp_servers"
+        package = importlib.import_module(package_name)
+        
+        # Find all modules in the package
+        for _, name, _ in pkgutil.iter_modules(package.__path__, package.__name__ + "."):
+            # Only process modules that start with "mcp_server_"
+            base_name = name.split(".")[-1]
+            if base_name.startswith("mcp_server_"):
+                # Generate a friendly server name
+                friendly_name = base_name[11:]
+                                
+                # Create env dictionary with placeholders
+                env_dict = {}
+                
+                # Create server configuration
+                config = {
+                    "command": "python",
+                    "args": ["-m", name],
+                    "env": env_dict
+                }                                
+                
+                # Store in dictionaries
+                servers[friendly_name] = config
+                templates[friendly_name] = json.dumps({friendly_name: config}, indent=4)
+                
+                logger.info(f"Detected MCP server: {friendly_name}")
+    
+    except Exception as e:
+        logger.error(f"Error generating server configs: {e}")
+    
+    return servers, templates
+
+
+# Automatically generate server configurations
+MCP_BUILD_IN_SERVERS, MCP_SERVER_TEMPLATES = _generate_server_configs()
 
 
 class McpHub:
@@ -422,3 +456,10 @@ class McpHub:
         """
         for name in list(self.connections.keys()):
             await self.delete_connection(name)
+
+    @classmethod
+    def get_server_templates(cls) -> Dict[str, str]:
+        """
+        Get all available server templates as JSON strings
+        """
+        return MCP_SERVER_TEMPLATES

autocoder/common/mcp_servers/mcp_server_gpt4o_mini_search.py

@@ -0,0 +1,153 @@
+from os import getenv
+from textwrap import dedent
+import sys
+
+import mcp.server.stdio
+import mcp.types as types
+from mcp.server import NotificationOptions, Server
+from mcp.server.models import InitializationOptions
+import json
+from openai import OpenAI
+
+OPENAI_API_KEY = getenv("OPENAI_API_KEY")
+# Check if API key is empty or None
+if not OPENAI_API_KEY:
+    print("Error: OPENAI_API_KEY environment variable is not set. Please set it before running this server.", file=sys.stderr)
+    sys.exit(1)
+
+OPENAI_API_BASE_URL = getenv(
+    "OPENAI_API_BASE_URL", "https://api.openai.com/v1")
+
+server = Server("mcp-server-gpt4o-mini-search")
+
+client = OpenAI(
+    api_key=OPENAI_API_KEY,
+    base_url=OPENAI_API_BASE_URL
+)
+
+
+@server.list_tools()
+async def handle_list_tools() -> list[types.Tool]:
+    return [
+        types.Tool(
+            name="gpt4o_mini_search",
+            description=dedent(
+                """
+                GPT-4o mini with search enables agents to gather information from the internet 
+                in real-time, providing up-to-date answers with source citations.
+                This tool is ideal for fact-checking, research, and accessing current information
+                that might not be in the model's training data.
+                
+                The search-enhanced responses include relevant web sources to support the information
+                provided, making it useful for obtaining verified and recent information.
+                
+                [Response structure]
+                - id: A unique identifier for the response
+                - model: The model used (gpt-4o-mini-search-preview)
+                - object: The object type ("chat.completion")
+                - created: The Unix timestamp when the completion was created
+                - choices[]: The list of completion choices generated
+                - usage: Usage statistics for the completion request
+                """
+            ),
+            inputSchema={
+                "type": "object",
+                "properties": {
+                    "system_message": {
+                        "type": "string",
+                        "description": "Optional custom system message. If not provided, a default search-optimized system message will be used.",
+                    },
+                    "messages": {
+                        "type": "array",
+                        "description": "A list of messages comprising the conversation so far (excluding system message which is handled separately).",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "content": {
+                                    "type": "string",
+                                    "description": "The contents of the message in this turn of conversation.",
+                                },
+                                "role": {
+                                    "type": "string",
+                                    "description": "The role of the speaker in this turn of conversation.",
+                                    "enum": ["user", "assistant"],
+                                },
+                            },
+                            "required": ["content", "role"],
+                        },
+                    },
+                },
+                "required": ["messages"],
+            },
+        )
+    ]
+
+
+@server.call_tool()
+async def handle_call_tool(
+    name: str, arguments: dict
+) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
+    if name != "gpt4o_mini_search":
+        raise ValueError(f"Unknown tool: {name}")
+
+    # Extract user messages
+    user_messages = arguments.get("messages", [])
+
+    # Define default system message if not provided
+    default_system_message = (
+        "你是专业搜索助手，需要：\n"
+        "1. 提供基于用户查询的清晰格式化信息\n"
+        "2. 使用[标题](URL)格式嵌入链接\n"
+        "3. 每条信息后附上来源\n"
+        "4. 用'---'分隔不同结果\n"
+        "5. 直接在文本中引用，不使用编号引用\n"
+        "6. 确保提供完整URL"
+    )
+
+    # Use custom system message if provided, otherwise use default
+    system_message = arguments.get("system_message", default_system_message)
+
+    # Prepare full message list with system message first
+    full_messages = [{"role": "system", "content": system_message}]
+    full_messages.extend(user_messages)
+
+    try:
+        # Initialize OpenAI client
+
+        # Make the API call using OpenAI SDK
+        completion = client.chat.completions.create(
+            model="gpt-4o-mini-search-preview",
+            messages=full_messages
+        )
+
+        # Extract content from response
+        content = completion.choices[0].message.content
+
+    except Exception as e:
+        raise RuntimeError(f"API error: {str(e)}")
+
+    return [types.TextContent(
+            type="text",
+            text=content,
+            )]
+
+
+async def main():
+    async with mcp.server.stdio.stdio_server() as (read_stream, write_stream):
+        await server.run(
+            read_stream,
+            write_stream,
+            InitializationOptions(
+                server_name="mcp-server-gpt4o-mini-search",
+                server_version="0.1.0",
+                capabilities=server.get_capabilities(
+                    notification_options=NotificationOptions(
+                        tools_changed=True),
+                    experimental_capabilities={},
+                ),
+            ),
+        )
+
+if __name__ == "__main__":
+    import asyncio
+    asyncio.run(main())

autocoder/rag/api_server.py

@@ -31,6 +31,14 @@ from byzerllm.utils.client.entrypoints.openai.protocol import (
 from pydantic import BaseModel
 from typing import List,Optional
 
+# If support dotenv, use it
+if os.path.exists(".env"):
+    try:
+        from dotenv import load_dotenv
+        load_dotenv()
+    except ImportError:
+        pass
+
 logger = init_logger(__name__)
 
 llm_client: ByzerLLM = None

autocoder/rag/cache/local_duckdb_storage_cache.py

@@ -0,0 +1,647 @@
+import hashlib
+import json
+import os
+import time
+import platform
+import threading
+from multiprocessing import Pool
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import List, Dict, Any, Optional, Tuple, Union
+import numpy as np
+from loguru import logger
+
+try:
+    import duckdb
+except ImportError:
+    logger.error("DuckDB is not installed, please install it using 'pip install duckdb'")
+    raise
+
+from autocoder.common import AutoCoderArgs
+from autocoder.common import SourceCode
+from autocoder.rag.cache.base_cache import (
+    BaseCacheManager,
+    DeleteEvent,
+    AddOrUpdateEvent,
+    FileInfo,
+    CacheItem
+)
+from autocoder.rag.utils import process_file_in_multi_process, process_file_local
+from autocoder.rag.variable_holder import VariableHolder
+from byzerllm import SimpleByzerLLM, ByzerLLM
+
+if platform.system() != "Windows":
+    import fcntl
+else:
+    fcntl = None
+
+
+default_ignore_dirs = [
+    "__pycache__",
+    "node_modules",
+    "_images"
+]
+
+
+def generate_file_md5(file_path: str) -> str:
+    md5_hash = hashlib.md5()
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            md5_hash.update(chunk)
+    return md5_hash.hexdigest()
+
+
+class DuckDBLocalContext:
+    def __init__(self, database_path: str):
+        self.database_path = database_path
+        self._conn = None
+
+    def _install_load_extension(self, ext_list):
+        for ext in ext_list:
+            self._conn.install_extension(ext)
+            self._conn.load_extension(ext)
+
+    def __enter__(self) -> "duckdb.DuckDBPyConnection":
+        if not os.path.exists(os.path.dirname(self.database_path)):
+            raise ValueError(
+                f"Directory {os.path.dirname(self.database_path)} does not exist."
+            )
+
+        self._conn = duckdb.connect(self.database_path)
+        self._install_load_extension(["json", "fts", "vss"])
+
+        return self._conn
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        if self._conn:
+            self._conn.close()
+
+
+class LocalDuckdbStorage:
+
+    def __init__(
+            self, llm: Union[ByzerLLM, SimpleByzerLLM] = None, database_name: str = ":memory:", table_name: str = "documents",
+            embed_dim: Optional[int] = None, persist_dir: str = "./storage"
+    ) -> None:
+        self.llm = llm
+        self.database_name = database_name
+        self.table_name = table_name
+        self.embed_dim = embed_dim
+        self.persist_dir = persist_dir
+        self.cache_dir = os.path.join(self.persist_dir, '.cache')
+        logger.info(f"正在启动 DuckDBVectorStore.")
+
+        if self.database_name != ":memory:":
+            self.database_path = os.path.join(self.cache_dir, self.database_name)
+
+        if self.database_name == ":memory:":
+            self._conn = duckdb.connect(self.database_name)
+            self._install_load_extension(["json", "fts", "vss"])
+            self._initialize()
+        else:
+            if not os.path.exists(self.database_path):
+                if not os.path.exists(self.cache_dir):
+                    os.makedirs(self.cache_dir)
+                self._initialize()
+            self._conn = None
+        logger.info(f"DuckDBVectorStore 初始化完成, 存储目录: {self.cache_dir}, "
+                    f"数据库名称: {self.database_name}, 数据表名称: {self.table_name}")
+
+    @classmethod
+    def class_name(cls) -> str:
+        return "DuckDBVectorStore"
+
+    @property
+    def client(self) -> Any:
+        """Return client."""
+        return self._conn
+
+    def _install_load_extension(self, ext_list):
+        for ext in ext_list:
+            self._conn.install_extension(ext)
+            self._conn.load_extension(ext)
+
+    @staticmethod
+    def _apply_pca(embedding, target_dim):
+        # 生成固定随机投影矩阵（避免每次调用重新生成）
+        np.random.seed(42)  # 固定随机种子保证一致性
+        source_dim = len(embedding)
+        projection_matrix = np.random.randn(source_dim, target_dim) / np.sqrt(source_dim)
+
+        # 执行投影
+        reduced = np.dot(embedding, projection_matrix)
+        return reduced
+
+    def _embedding(self, context: str, norm: bool = True, dim: int | None = None) -> List[float]:
+        embedding = self.llm.emb_query(context)[0].output
+
+        if dim:
+            embedding = self._apply_pca(embedding, target_dim=dim)  # 降维后形状 (1024,)
+
+        if norm:
+            embedding = embedding / np.linalg.norm(embedding)
+
+        return embedding.tolist()
+
+    def _initialize(self) -> None:
+        if self.embed_dim is None:
+            _query = f"""
+                CREATE TABLE IF NOT EXISTS {self.table_name} (
+                    _id VARCHAR,
+                    file_path VARCHAR,
+                    content TEXT,
+                    raw_content TEXT,
+                    vector FLOAT[],
+                    mtime FLOAT
+                );
+            """
+        else:
+            _query = f"""
+                CREATE TABLE IF NOT EXISTS {self.table_name} (
+                    _id VARCHAR,
+                    file_path VARCHAR,
+                    content TEXT,
+                    raw_content TEXT,
+                    vector FLOAT[],
+                    mtime FLOAT
+                );
+            """
+
+        if self.database_name == ":memory:":
+            self._conn.execute(_query)
+        elif self.database_path is not None:
+            with DuckDBLocalContext(self.database_path) as _conn:
+                _conn.execute(_query)
+
+    def truncate_table(self):
+        _truncate_query = f"""TRUNCATE TABLE {self.table_name};"""
+        if self.database_name == ":memory:":
+            self._conn.execute(_truncate_query)
+        elif self.database_path is not None:
+            with DuckDBLocalContext(self.database_path) as _conn:
+                _conn.execute(_truncate_query)
+
+    def query_by_path(self, file_path: str):
+        _exists_query = f"""SELECT _id FROM {self.table_name} WHERE file_path = ?"""
+        query_params = [
+            file_path
+        ]
+        _final_results = []
+        if self.database_name == ":memory:":
+            _final_results = self._conn.execute(_exists_query, query_params).fetchall()
+        elif self.database_path is not None:
+            with DuckDBLocalContext(self.database_path) as _conn:
+                _final_results = _conn.execute(_exists_query, query_params).fetchall()
+        return _final_results
+
+    def delete_by_ids(self, _ids: List[str]):
+        _delete_query = f"""DELETE FROM {self.table_name} WHERE _id IN (?);"""
+        query_params = [
+            ','.join(_ids)
+        ]
+        if self.database_name == ":memory:":
+            _final_results = self._conn.execute(_delete_query, query_params).fetchall()
+        elif self.database_path is not None:
+            with DuckDBLocalContext(self.database_path) as _conn:
+                _final_results = _conn.execute(_delete_query, query_params).fetchall()
+        return _final_results
+
+    def _node_to_table_row(self, context_chunk: Dict[str, str | float], dim: int | None = None) -> Any:
+        return (
+            context_chunk["_id"],
+            context_chunk["file_path"],
+            context_chunk["content"],
+            context_chunk["raw_content"],
+            self._embedding(context_chunk["raw_content"], norm=True, dim=dim),
+            context_chunk["mtime"]
+        )
+
+    def add_doc(self, context_chunk: Dict[str, str | float], dim: int | None = None):
+        """
+        {
+            "_id": f"{doc.module_name}_{chunk_idx}",
+            "file_path": file_info.file_path,
+            "content": chunk,
+            "raw_content": chunk,
+            "vector": chunk,
+            "mtime": file_info.modify_time,
+        }
+        """
+        if self.database_name == ":memory:":
+            _table = self._conn.table(self.table_name)
+            _row = self._node_to_table_row(context_chunk, dim=dim)
+            _table.insert(_row)
+        elif self.database_path is not None:
+            with DuckDBLocalContext(self.database_path) as _conn:
+                _table = _conn.table(self.table_name)
+                _row = self._node_to_table_row(context_chunk, dim=dim)
+                _table.insert(_row)
+
+    def vector_search(
+            self, query: str, similarity_value: float = 0.7, similarity_top_k: int = 10, query_dim: int | None = None
+    ):
+        """
+        list_cosine_similarity: 计算两个列表之间的余弦相似度
+        list_cosine_distance: 计算两个列表之间的余弦距离
+        list_dot_product: 计算两个大小相同的数字列表的点积
+        """
+        _db_query = f"""
+            SELECT _id, file_path, mtime, score
+            FROM (
+                SELECT *, list_cosine_similarity(vector, ?) AS score
+                FROM {self.table_name}
+            ) sq
+            WHERE score IS NOT NULL
+            AND score >= ?
+            ORDER BY score DESC LIMIT ?;
+        """
+        query_params = [
+            self._embedding(query, norm=True, dim=query_dim),
+            similarity_value,
+            similarity_top_k,
+        ]
+
+        _final_results = []
+        if self.database_name == ":memory:":
+            _final_results = self._conn.execute(_db_query, query_params).fetchall()
+        elif self.database_path is not None:
+            with DuckDBLocalContext(self.database_path) as _conn:
+                _final_results = _conn.execute(_db_query, query_params).fetchall()
+        return _final_results
+
+
+efault_ignore_dirs = [
+    "__pycache__",
+    "node_modules",
+    "_images"
+]
+
+
+class LocalDuckDBStorageCache(BaseCacheManager):
+    def __init__(
+            self,
+            path,
+            ignore_spec,
+            required_exts,
+            extra_params: Optional[AutoCoderArgs] = None,
+            emb_llm: Union[ByzerLLM, SimpleByzerLLM] = None
+    ):
+        self.path = path
+        self.ignore_spec = ignore_spec
+        self.required_exts = required_exts
+        self.extra_params = extra_params
+
+        self.storage = LocalDuckdbStorage(
+            llm=emb_llm,
+            database_name="byzerai_store_duckdb.db",
+            table_name="rag_duckdb",
+            persist_dir=self.path
+        )
+        self.queue = []
+        self.chunk_size = 1000
+        self.max_output_tokens = extra_params.hybrid_index_max_output_tokens
+
+        # 设置缓存文件路径
+        self.cache_dir = os.path.join(self.path, ".cache")
+        self.cache_file = os.path.join(self.cache_dir, "duckdb_storage_speedup.jsonl")
+        self.cache: Dict[str, CacheItem] = {}
+        # 创建缓存目录
+        if not os.path.exists(self.cache_dir):
+            os.makedirs(self.cache_dir)
+
+        self.lock = threading.Lock()
+        self.stop_event = threading.Event()
+        self.thread = threading.Thread(target=self.process_queue)
+        self.thread.daemon = True
+        self.thread.start()
+
+        # 加载缓存
+        self.cache = self._load_cache()
+
+    @staticmethod
+    def _chunk_text(text, max_length=1000):
+        """Split text into chunks"""
+        chunks = []
+        current_chunk = []
+        current_length = 0
+
+        for line in text.split("\n"):
+            if current_length + len(line) > max_length and current_chunk:
+                chunks.append("\n".join(current_chunk))
+                current_chunk = []
+                current_length = 0
+            current_chunk.append(line)
+            current_length += len(line)
+
+        if current_chunk:
+            chunks.append("\n".join(current_chunk))
+
+        return chunks
+
+    def _load_cache(self) -> Dict[str, CacheItem]:
+        """Load cache from file"""
+        if os.path.exists(self.cache_file):
+            try:
+                with open(self.cache_file, "r", encoding="utf-8") as f:
+                    lines = f.readlines()
+                    cache = {}
+                    for line in lines:
+                        try:
+                            data = json.loads(line.strip())
+                            if isinstance(data, dict) and "file_path" in data:
+                                # 转换为 CacheItem 对象
+                                cache_item = CacheItem.model_validate(data)
+                                cache[data["file_path"]] = cache_item
+                        except json.JSONDecodeError:
+                            continue
+                    return cache
+            except Exception as e:
+                logger.error(f"Error loading cache file: {str(e)}")
+                return {}
+        return {}
+
+    def write_cache(self):
+        cache_file = self.cache_file
+
+        if not fcntl:
+            try:
+                with open(cache_file, "w", encoding="utf-8") as f:
+                    for cache_item in self.cache.values():
+                        # 确保序列化 Pydantic 模型
+                        json.dump(cache_item.model_dump(), f, ensure_ascii=False)
+                        f.write("\n")
+            except IOError as e:
+                logger.error(f"Error writing cache file: {str(e)}")
+        else:
+            lock_file = cache_file + ".lock"
+            with open(lock_file, "w", encoding="utf-8") as lockf:
+                try:
+                    # 获取文件锁
+                    fcntl.flock(lockf, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                    # 写入缓存文件
+                    with open(cache_file, "w", encoding="utf-8") as f:
+                        for cache_item in self.cache.values():
+                            # 确保序列化 Pydantic 模型
+                            json.dump(cache_item.model_dump(), f, ensure_ascii=False)
+                            f.write("\n")
+
+                finally:
+                    # 释放文件锁
+                    fcntl.flock(lockf, fcntl.LOCK_UN)
+
+    @staticmethod
+    def fileinfo_to_tuple(file_info: FileInfo) -> Tuple[str, str, float, str]:
+        return file_info.file_path, file_info.relative_path, file_info.modify_time, file_info.file_md5
+
+    def build_cache(self):
+        """Build the cache by reading files and storing in DuckDBVectorStore"""
+        logger.info(f"Building cache for path: {self.path}")
+
+        files_to_process = []
+        for file_info in self.get_all_files():
+            if (
+                    file_info.file_path not in self.cache
+                    or self.cache[file_info.file_path].md5 != file_info.file_md5
+            ):
+                files_to_process.append(file_info)
+
+        if not files_to_process:
+            return
+
+        from autocoder.rag.token_counter import initialize_tokenizer
+
+        with Pool(
+            processes=os.cpu_count(),
+            initializer=initialize_tokenizer,
+            initargs=(VariableHolder.TOKENIZER_PATH,),
+        ) as pool:
+            target_files_to_process = []
+            for file_info in files_to_process:
+                target_files_to_process.append(
+                    self.fileinfo_to_tuple(file_info))
+            results = pool.map(process_file_in_multi_process,
+                               target_files_to_process)
+
+        items = []
+        for file_info, result in zip(files_to_process, results):
+            content: List[SourceCode] = result
+            self.cache[file_info.file_path] = CacheItem(
+                file_path=file_info.file_path,
+                relative_path=file_info.relative_path,
+                content=[c.model_dump() for c in content],
+                modify_time=file_info.modify_time,
+                md5=file_info.file_md5,
+            )
+
+            for doc in content:
+                logger.info(f"Processing file: {doc.module_name}")
+                chunks = self._chunk_text(doc.source_code, self.chunk_size)
+                for chunk_idx, chunk in enumerate(chunks):
+                    chunk_item = {
+                        "_id": f"{doc.module_name}_{chunk_idx}",
+                        "file_path": file_info.file_path,
+                        "content": chunk,
+                        "raw_content": chunk,
+                        "vector": "",
+                        "mtime": file_info.modify_time,
+                    }
+                    items.append(chunk_item)
+
+        # Save to local cache
+        logger.info("Saving cache to local file")
+        self.write_cache()
+
+        if items:
+            logger.info("Clear cache from Byzer DuckDB Storage")
+            self.storage.truncate_table()
+            logger.info("Save new cache to Byzer DuckDB Storage")
+
+            total_chunks = len(items)
+            completed_chunks = 0
+
+            logger.info(f"进度: 已完成 {0}/{total_chunks} 个文本块")
+
+            for _chunk in items:
+                try:
+                    self.storage.add_doc(_chunk, dim=self.extra_params.rag_duckdb_vector_dim)
+                    completed_chunks += 1
+                    logger.info(f"进度: 已完成 {completed_chunks}/{total_chunks} 个文本块")
+                    time.sleep(self.extra_params.anti_quota_limit)
+                except Exception as err:
+                    logger.error(f"Error in saving chunk: {str(err)}")
+
+    def update_storage(self, file_info: FileInfo, is_delete: bool):
+        results = self.storage.query_by_path(file_info.file_path)
+        if results:  # [('_id',)]
+            for result in results:
+                self.storage.delete_by_ids([result[0]])
+
+        items = []
+        if not is_delete:
+            content = [
+                SourceCode.model_validate(doc) for doc in self.cache[file_info.file_path].content
+            ]
+            modify_time = self.cache[file_info.file_path].modify_time
+            for doc in content:
+                logger.info(f"正在处理更新文件: {doc.module_name}")
+                chunks = self._chunk_text(doc.source_code, self.chunk_size)
+                for chunk_idx, chunk in enumerate(chunks):
+                    chunk_item = {
+                        "_id": f"{doc.module_name}_{chunk_idx}",
+                        "file_path": file_info.file_path,
+                        "content": chunk,
+                        "raw_content": chunk,
+                        "vector": chunk,
+                        "mtime": modify_time,
+                    }
+                    items.append(chunk_item)
+        if items:
+            for _chunk in items:
+                try:
+                    self.storage.add_doc(_chunk, dim=self.extra_params.rag_duckdb_vector_dim)
+                    time.sleep(self.extra_params.anti_quota_limit)
+                except Exception as err:
+                    logger.error(f"Error in saving chunk: {str(err)}")
+
+    def process_queue(self):
+        while self.queue:
+            file_list = self.queue.pop(0)
+            if isinstance(file_list, DeleteEvent):
+                for item in file_list.file_paths:
+                    logger.info(f"{item} is detected to be removed")
+                    del self.cache[item]
+                    # 创建一个临时的 FileInfo 对象
+                    file_info = FileInfo(
+                        file_path=item, relative_path="", modify_time=0, file_md5="")
+                    self.update_storage(file_info, is_delete=True)
+
+            elif isinstance(file_list, AddOrUpdateEvent):
+                for file_info in file_list.file_infos:
+                    logger.info(
+                        f"{file_info.file_path} is detected to be updated")
+                    # 处理文件并创建 CacheItem
+                    # content = process_file_local(
+                    #     self.fileinfo_to_tuple(file_info))
+                    content = process_file_local(file_info.file_path)
+                    self.cache[file_info.file_path] = CacheItem(
+                        file_path=file_info.file_path,
+                        relative_path=file_info.relative_path,
+                        content=[c.model_dump() for c in content],
+                        modify_time=file_info.modify_time,
+                        md5=file_info.file_md5,
+                    )
+                    self.update_storage(file_info, is_delete=False)
+            self.write_cache()
+
+    def trigger_update(self):
+        logger.info("检查文件是否有更新.....")
+        files_to_process = []
+        current_files = set()
+        for file_info in self.get_all_files():
+            current_files.add(file_info.file_path)
+            if (
+                    file_info.file_path not in self.cache
+                    or self.cache[file_info.file_path].md5 != file_info.file_md5
+            ):
+                files_to_process.append(file_info)
+
+        deleted_files = set(self.cache.keys()) - current_files
+        logger.info(f"待处理的文件: {len(files_to_process)}个")
+        logger.info(f"已删除的文件: {len(deleted_files)}个")
+        if deleted_files:
+            with self.lock:
+                self.queue.append(DeleteEvent(file_paths=deleted_files))
+        if files_to_process:
+            with self.lock:
+                self.queue.append(AddOrUpdateEvent(file_infos=files_to_process))
+
+    def get_all_files(self) -> List[FileInfo]:
+        all_files = []
+        for root, dirs, files in os.walk(self.path, followlinks=True):
+            dirs[:] = [d for d in dirs if not d.startswith(
+                ".") and d not in default_ignore_dirs]
+
+            if self.ignore_spec:
+                relative_root = os.path.relpath(root, self.path)
+                dirs[:] = [
+                    d
+                    for d in dirs
+                    if not self.ignore_spec.match_file(os.path.join(relative_root, d))
+                ]
+                files = [
+                    f
+                    for f in files
+                    if not self.ignore_spec.match_file(os.path.join(relative_root, f))
+                ]
+
+            for file in files:
+                if self.required_exts and not any(
+                        file.endswith(ext) for ext in self.required_exts
+                ):
+                    continue
+
+                file_path = os.path.join(root, file)
+                relative_path = os.path.relpath(file_path, self.path)
+                modify_time = os.path.getmtime(file_path)
+                file_md5 = generate_file_md5(file_path)
+                all_files.append(
+                    FileInfo(
+                        file_path=file_path,
+                        relative_path=relative_path,
+                        modify_time=modify_time,
+                        file_md5=file_md5))
+
+        return all_files
+
+    def get_cache(self, options: Optional[Dict[str, Any]] = None) -> Dict[str, Dict]:
+        """Search cached documents using query"""
+        self.trigger_update()  # 检查更新
+
+        if options is None or "query" not in options:
+            return {file_path: self.cache[file_path].model_dump() for file_path in self.cache}
+
+        query = options.get("query", "")
+        logger.info(f"正在使用向量搜索检索数据, 你的问题: {query}")
+        total_tokens = 0
+        results = []
+
+        # Add vector search if enabled
+        if options.get("enable_vector_search", True):
+            # 返回值包含  [(_id, file_path, mtime, score,),]
+            # results = self.storage.vector_search(query, similarity_value=0.7, similarity_top_k=200)
+            search_results = self.storage.vector_search(
+                query,
+                similarity_value=self.extra_params.duckdb_query_similarity,
+                similarity_top_k=self.extra_params.duckdb_query_top_k,
+                query_dim=self.extra_params.duckdb_vector_dim
+            )
+            results.extend(search_results)
+
+        # Group results by file_path and reconstruct documents while preserving order
+        # 这里还可以有排序优化，综合考虑一篇内容出现的次数以及排序位置
+        file_paths = []
+        seen = set()
+        for result in results:
+            _id, _file_path, _mtime, _score = result
+            if _file_path not in seen:
+                seen.add(_file_path)
+                file_paths.append(_file_path)
+
+        # 从缓存中获取文件内容
+        result = {}
+        for file_path in file_paths:
+            if file_path in self.cache:
+                cached_data = self.cache[file_path]
+                for doc in cached_data.content:
+                    if total_tokens + doc["tokens"] > self.max_output_tokens:
+                        logger.info(
+                            f"当前检索已超出用户设置 Hybrid Index Max Tokens:{self.max_output_tokens}，"
+                            f"累计tokens: {total_tokens}, "
+                            f"经过向量搜索共检索出 {len(result.keys())} 个文档, 共 {len(self.cache.keys())} 个文档")
+                        return result
+                    total_tokens += doc["tokens"]
+                result[file_path] = cached_data.model_dump()
+        logger.info(
+            f"用户Hybrid Index Max Tokens设置为:{self.max_output_tokens}，"
+            f"累计tokens: {total_tokens}, "
+            f"经过向量搜索共检索出 {len(result.keys())} 个文档, 共 {len(self.cache.keys())} 个文档")
+        return result

autocoder/rag/document_retriever.py

@@ -11,6 +11,7 @@ from autocoder.rag.cache.simple_cache import AutoCoderRAGAsyncUpdateQueue
 from autocoder.rag.cache.file_monitor_cache import AutoCoderRAGDocListener
 from autocoder.rag.cache.byzer_storage_cache import ByzerStorageCache
 from autocoder.rag.cache.local_byzer_storage_cache import LocalByzerStorageCache
+from autocoder.rag.cache.local_duckdb_storage_cache import LocalDuckDBStorageCache
 from autocoder.common import AutoCoderArgs
 
 cache_lock = threading.Lock()
@@ -66,11 +67,17 @@ class LocalDocumentRetriever(BaseDocumentRetriever):
                 self.cacher = ByzerStorageCache(
                     path, ignore_spec, required_exts, extra_params
                 )
-            else:                                
-                self.cacher = LocalByzerStorageCache(
-                    path, ignore_spec, required_exts, extra_params,
-                    emb_llm = emb_llm
-                )
+            else:
+                if extra_params.rag_storage_type == "duckdb":
+                    self.cacher = LocalDuckDBStorageCache(
+                        path, ignore_spec, required_exts, extra_params,
+                        emb_llm=emb_llm
+                    )
+                elif extra_params.rag_storage_type in ["byzer-storage", "byzer_storage"]:
+                    self.cacher = LocalByzerStorageCache(
+                        path, ignore_spec, required_exts, extra_params,
+                        emb_llm=emb_llm
+                    )
         else:
             if self.monitor_mode:
                 self.cacher = AutoCoderRAGDocListener(

autocoder/rag/long_context_rag.py

@@ -210,22 +210,22 @@ class LongContextRAG:
 
         avg_tokens = statistics.mean(token_counts) if token_counts else 0
         median_tokens = statistics.median(token_counts) if token_counts else 0
-
-        logger.info(
-            "RAG Configuration:\n"
-            f"  Total docs:        {doc_num}\n"
-            f"  Total tokens:      {token_num}\n"
-            f"  Tokenizer path:    {self.tokenizer_path}\n"
-            f"  Relevant score:    {self.relevant_score}\n"
-            f"  Token limit:       {self.token_limit}\n"
-            f"  Full text limit:   {self.full_text_limit}\n"
-            f"  Segment limit:     {self.segment_limit}\n"
-            f"  Buff limit:        {self.buff_limit}\n"
-            f"  Max doc tokens:    {max(token_counts) if token_counts else 0}\n"
-            f"  Min doc tokens:    {min(token_counts) if token_counts else 0}\n"
-            f"  Avg doc tokens:    {avg_tokens:.2f}\n"
-            f"  Median doc tokens: {median_tokens:.2f}\n"
-        )
+        if not self.client:
+            logger.info(
+                "RAG Configuration:\n"
+                f"  Total docs:        {doc_num}\n"
+                f"  Total tokens:      {token_num}\n"
+                f"  Tokenizer path:    {self.tokenizer_path}\n"
+                f"  Relevant score:    {self.relevant_score}\n"
+                f"  Token limit:       {self.token_limit}\n"
+                f"  Full text limit:   {self.full_text_limit}\n"
+                f"  Segment limit:     {self.segment_limit}\n"
+                f"  Buff limit:        {self.buff_limit}\n"
+                f"  Max doc tokens:    {max(token_counts) if token_counts else 0}\n"
+                f"  Min doc tokens:    {min(token_counts) if token_counts else 0}\n"
+                f"  Avg doc tokens:    {avg_tokens:.2f}\n"
+                f"  Median doc tokens: {median_tokens:.2f}\n"
+            )
 
     def count_tokens(self, text: str) -> int:
         if self.tokenizer is None:

autocoder/version.py

@@ -1 +1 @@
-__version__ = "0.1.291"
+__version__ = "0.1.293"