auto-coder 0.1.232__py3-none-any.whl → 0.1.233__py3-none-any.whl

{auto_coder-0.1.232.dist-info → auto_coder-0.1.233.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.232
+Version: 0.1.233
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence

{auto_coder-0.1.232.dist-info → auto_coder-0.1.233.dist-info}/RECORD

@@ -1,17 +1,17 @@
 autocoder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-autocoder/auto_coder.py,sha256=REnZJUy4Dx9b7Z8gM9D4SWMkbHXl0JjF-DFiod4aB-A,58357
+autocoder/auto_coder.py,sha256=r2uHo-xPDbel6OhZwGzl8Iz1GP4d9KUWcDReEol9LOQ,59541
 autocoder/auto_coder_lang.py,sha256=Rtupq6N3_HT7JRhDKdgCBcwRaiAnyCOR_Gsp4jUomrI,3229
 autocoder/auto_coder_rag.py,sha256=illKgzP2bv-Tq50ujsofJnOHdI4pzr0ALtfR8NHHWdQ,22351
 autocoder/auto_coder_rag_client_mcp.py,sha256=WV7j5JUiQge0x4-B7Hp5-pSAFXLbvLpzQMcCovbauIM,6276
 autocoder/auto_coder_rag_mcp.py,sha256=-RrjNwFaS2e5v8XDIrKR-zlUNUE8UBaeOtojffBrvJo,8521
 autocoder/auto_coder_server.py,sha256=XU9b4SBH7zjPPXaTWWHV4_zJm-XYa6njuLQaplYJH_c,20290
 autocoder/benchmark.py,sha256=Ypomkdzd1T3GE6dRICY3Hj547dZ6_inqJbBJIp5QMco,4423
-autocoder/chat_auto_coder.py,sha256=KtDAwIiBB1b2jBSY8BCoSj88iRSwtRACkzME9h91ido,101601
+autocoder/chat_auto_coder.py,sha256=PzrbhpwTgJEMuPOT7vBd4uPi58mwqLc59l2fVV6rJc8,102049
 autocoder/chat_auto_coder_lang.py,sha256=YJsFi8an0Kjbo9X7xKZfpdbHS3rbhrvChZNjWqEQ5Sw,11032
 autocoder/command_args.py,sha256=9aYJ-AmPxP1sQh6ciw04FWHjSn31f2W9afXFwo8wgx4,30441
 autocoder/lang.py,sha256=U6AjVV8Rs1uLyjFCZ8sT6WWuNUxMBqkXXIOs4S120uk,14511
 autocoder/models.py,sha256=FlBrF6HhGao_RiCSgYhCmP7vs0KlG4hI_BI6dyZiL9s,5292
-autocoder/version.py,sha256=pvV6K1v2HfuQywwyaRYD8uMohj-o0EfkRauHjMIrYtk,24
+autocoder/version.py,sha256=VphyhuzLuUXUhi7WwvwaGVGg2OSz77iY97Prah1F5g8,24
 autocoder/agent/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/agent/auto_demand_organizer.py,sha256=NWSAEsEk94vT3lGjfo25kKLMwYdPcpy9e-i21txPasQ,6942
 autocoder/agent/auto_filegroup.py,sha256=CW7bqp0FW1GIEMnl-blyAc2UGT7O9Mom0q66ITz1ckM,6635
@@ -24,7 +24,7 @@ autocoder/agent/project_reader.py,sha256=tWLaPoLw1gI6kO_NzivQj28KbobU2ceOLuppHMb
 autocoder/chat/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/common/JupyterClient.py,sha256=O-wi6pXeAEYhAY24kDa0BINrLYvKS6rKyWe98pDClS0,2816
 autocoder/common/ShellClient.py,sha256=fM1q8t_XMSbLBl2zkCNC2J9xuyKN3eXzGm6hHhqL2WY,2286
-autocoder/common/__init__.py,sha256=-ATCQp7EhFepyUsupHEZpUN3ouQ-OcM3PAoCnAXezhg,11652
+autocoder/common/__init__.py,sha256=JDIZ_1JP7Nm74OL_aCFwvhiwiynukaE5oPCfh5GPQWU,11695
 autocoder/common/anything2images.py,sha256=0ILBbWzY02M-CiWB-vzuomb_J1hVdxRcenAfIrAXq9M,25283
 autocoder/common/anything2img.py,sha256=4TREa-sOA-iargieUy7MpyCYVUE-9Mmq0wJtwomPqnE,7662
 autocoder/common/audio.py,sha256=Kn9nWKQddWnUrAz0a_ZUgjcu4VUU_IcZBigT7n3N3qc,7439
@@ -69,16 +69,20 @@ autocoder/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/db/store.py,sha256=tFT66bP2ZKIqZip-uhLkHRSLaaOAUUDZfozJwcqix3c,1908
 autocoder/dispacher/__init__.py,sha256=YoA64dIxnx4jcE1pwSfg81sjkQtjDkhddkfac1-cMWo,1230
 autocoder/dispacher/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-autocoder/dispacher/actions/action.py,sha256=JqghsBpgrM-5J79UYjGIVivNgz6YnMtibwBPP0Ww9xY,19299
-autocoder/dispacher/actions/copilot.py,sha256=BdD-L0LFb5YUH2C87IVCm446SJLyOltaaG9shmsQ25Q,13072
+autocoder/dispacher/actions/action.py,sha256=NjJGLek8H0FlIOreBnl2KEC-jJ5Jq-V8D1RuI6ifUjc,19299
+autocoder/dispacher/actions/copilot.py,sha256=iMh4ckj9hO5Q-iemF3CStXd7DatWai7Eci5zOlKxK9c,13072
 autocoder/dispacher/actions/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-autocoder/dispacher/actions/plugins/action_regex_project.py,sha256=oldAZeGaDcGQBiTdCigO6yiV_GJQTGRLy_B_hVHAQZw,5325
+autocoder/dispacher/actions/plugins/action_regex_project.py,sha256=-ly-NRgQ8LfDQDoH0QmD_LP-G932Kt08WXy9oIvLy10,5325
 autocoder/dispacher/actions/plugins/action_translate.py,sha256=nVAtRSQpdGNmZxg1R_9zXG3AuTv3CHf2v7ODgj8u65c,7727
 autocoder/index/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+autocoder/index/entry.py,sha256=1T41clV3GXwiRbowubQ1iZM5k3_2ECS-DtcnbA9QxAk,10081
 autocoder/index/for_command.py,sha256=LGnz-OWogT8rd24m4Zcan7doLaijxqorAuiMk7WuRq0,3125
-autocoder/index/index.py,sha256=1HLwK-ylpibYjHgPommdaL1bRwGP1QGWONaNYExzqRc,34828
+autocoder/index/index.py,sha256=lwaobSHvOnzhTMf8SQXzw3nIJQUS4lyo6nLdtv0Ebc0,19223
 autocoder/index/symbols_utils.py,sha256=CjcjUVajmJZB75Ty3a7kMv1BZphrm-tIBAdOJv6uo-0,2037
-autocoder/index/types.py,sha256=tYoFicbS6k1Dx4EoMpuNq71-4pF6hhEbtej0VYCVlSo,524
+autocoder/index/types.py,sha256=a2s_KV5FJlq7jqA2ELSo9E1sjuLwDB-JJYMhSpzBAhU,596
+autocoder/index/filter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+autocoder/index/filter/normal_filter.py,sha256=pE5QwcBq6NYHFtYhwhfMJmYQYJwErNs-Q7iZmVBAh-k,7964
+autocoder/index/filter/quick_filter.py,sha256=BxOiZOlK2v6EnX0yV28R3ielXboTmrMvVwielCrqKpE,3678
 autocoder/pyproject/__init__.py,sha256=dQ2_7YZ7guybT9BhfxSGn43eLQJGQN2zgeKa6--JlaQ,14403
 autocoder/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/rag/api_server.py,sha256=dRbhAZVRAOlZ64Cnxf4_rKb4iJwHnrWS9Zr67IVORw0,7288
@@ -133,9 +137,9 @@ autocoder/utils/tests.py,sha256=BqphrwyycGAvs-5mhH8pKtMZdObwhFtJ5MC_ZAOiLq8,1340
 autocoder/utils/auto_coder_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 autocoder/utils/auto_coder_utils/chat_stream_out.py,sha256=6D_SIa5hHSwIHC1poO_ztK7IVugAqNHu-jQySd7EnfQ,4181
 autocoder/utils/chat_auto_coder_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-auto_coder-0.1.232.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-auto_coder-0.1.232.dist-info/METADATA,sha256=5XhWylx2N9N4DNDoJpF3GUWHJzpNVg2wwgBZ5m_rBDU,2641
-auto_coder-0.1.232.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-auto_coder-0.1.232.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
-auto_coder-0.1.232.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
-auto_coder-0.1.232.dist-info/RECORD,,
+auto_coder-0.1.233.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+auto_coder-0.1.233.dist-info/METADATA,sha256=w___EO9VOCh1T58HsPM3Qi5OYm2ktUVVD6I5KucBO3w,2641
+auto_coder-0.1.233.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+auto_coder-0.1.233.dist-info/entry_points.txt,sha256=0nzHtHH4pNcM7xq4EBA2toS28Qelrvcbrr59GqD_0Ak,350
+auto_coder-0.1.233.dist-info/top_level.txt,sha256=Jqc0_uJSw2GwoFQAa9iJxYns-2mWla-9ok_Y3Gcznjk,10
+auto_coder-0.1.233.dist-info/RECORD,,

autocoder/auto_coder.py

@@ -42,6 +42,7 @@ from rich.live import Live
 from autocoder.auto_coder_lang import get_message
 from autocoder.common.memory_manager import save_to_memory_file
 from autocoder import models as models_module
+from autocoder.common.utils_code_auto_generate import stream_chat_with_continue
 from autocoder.utils.auto_coder_utils.chat_stream_out import stream_out
 
 console = Console()
@@ -443,7 +444,24 @@ def main(input_args: Optional[List[str]] = None):
                         "saas.is_reasoning": model_info["is_reasoning"]
                     }
                 )
-                llm.setup_sub_client("inference_model", inference_model)        
+                llm.setup_sub_client("inference_model", inference_model) 
+
+            if args.index_filter_model:
+                model_info = models_module.get_model_by_name(args.index_filter_model)
+                model_name = args.index_filter_model
+                index_filter_model = byzerllm.SimpleByzerLLM(default_model_name=model_name)
+                index_filter_model.deploy(
+                    model_path="",
+                    pretrained_model_type=model_info["model_type"],
+                    udf_name=model_name,
+                    infer_params={
+                        "saas.base_url": model_info["base_url"],
+                        "saas.api_key": model_info["api_key"],
+                        "saas.model": model_info["model_name"],
+                        "saas.is_reasoning": model_info["is_reasoning"]
+                    }
+                )
+                llm.setup_sub_client("index_filter_model", index_filter_model)            
 
 
         if args.product_mode == "pro":
@@ -482,7 +500,12 @@ def main(input_args: Optional[List[str]] = None):
             if args.inference_model:
                 inference_model = byzerllm.ByzerLLM()
                 inference_model.setup_default_model_name(args.inference_model)
-                llm.setup_sub_client("inference_model", inference_model)        
+                llm.setup_sub_client("inference_model", inference_model)   
+
+            if args.index_filter_model:
+                index_filter_model = byzerllm.ByzerLLM()
+                index_filter_model.setup_default_model_name(args.index_filter_model)
+                llm.setup_sub_client("index_filter_model", index_filter_model)
             
 
         if args.human_as_model:
@@ -1106,7 +1129,8 @@ def main(input_args: Optional[List[str]] = None):
                         {"role": "assistant", "content": "read"})
                     source_count += 1
 
-            from autocoder.index.index import IndexManager, build_index_and_filter_files
+            from autocoder.index.index import IndexManager
+            from autocoder.index.entry import build_index_and_filter_files
             from autocoder.pyproject import PyProject
             from autocoder.tsproject import TSProject
             from autocoder.suffixproject import SuffixProject
@@ -1272,8 +1296,7 @@ def main(input_args: Optional[List[str]] = None):
                     )
                 )
                 v = [[response.result,None]]
-            else:
-                from autocoder.common.utils_code_auto_generate import stream_chat_with_continue
+            else:                
                 v = stream_chat_with_continue(
                     llm=chat_llm,
                     conversations=loaded_conversations,

autocoder/chat_auto_coder.py

@@ -52,6 +52,7 @@ from autocoder.common.memory_manager import get_global_memory_file_paths
 from autocoder import models
 import shlex
 from autocoder.utils.llms import get_single_llm
+import pkg_resources
 
 class SymbolItem(BaseModel):
     symbol_name: str
@@ -2478,6 +2479,17 @@ def lib_command(args: List[str]):
 
 
 def main():
+    from autocoder.rag.variable_holder import VariableHolder
+    from tokenizers import Tokenizer    
+    try:
+        tokenizer_path = pkg_resources.resource_filename(
+            "autocoder", "data/tokenizer.json"
+        )
+        VariableHolder.TOKENIZER_PATH = tokenizer_path
+        VariableHolder.TOKENIZER_MODEL = Tokenizer.from_file(tokenizer_path)
+    except FileNotFoundError:
+        tokenizer_path = None
+        
     ARGS = parse_arguments()
     
     if ARGS.lite:

autocoder/common/__init__.py

@@ -259,6 +259,7 @@ class AutoCoderArgs(pydantic.BaseModel):
     skip_filter_index: Optional[bool] = False
 
     index_model: Optional[str] = ""
+    index_filter_model: Optional[str] = ""
     index_model_max_length: Optional[int] = 0
     index_model_max_input_length: Optional[int] = 0
     index_model_anti_quota_limit: Optional[int] = 0

autocoder/dispacher/actions/action.py

@@ -9,7 +9,7 @@ from autocoder.common.buildin_tokenizer import BuildinTokenizer
 from autocoder.pyproject import PyProject, Level1PyProject
 from autocoder.tsproject import TSProject
 from autocoder.suffixproject import SuffixProject
-from autocoder.index.index import build_index_and_filter_files
+from autocoder.index.entry import build_index_and_filter_files
 from autocoder.common.code_auto_merge import CodeAutoMerge
 from autocoder.common.code_auto_merge_diff import CodeAutoMergeDiff
 from autocoder.common.code_auto_merge_strict_diff import CodeAutoMergeStrictDiff

autocoder/dispacher/actions/copilot.py

@@ -9,7 +9,7 @@ from autocoder.common.JupyterClient import JupyterNotebook
 from autocoder.common.ShellClient import ShellClient
 from autocoder.suffixproject import SuffixProject
 from autocoder.common.search import Search, SearchEngine
-from autocoder.index.index import build_index_and_filter_files
+from autocoder.index.entry import build_index_and_filter_files
 from autocoder.common.image_to_page import ImageToPage, ImageToPageDirectly
 from typing import Optional, Dict, Any, List
 import byzerllm

autocoder/dispacher/actions/plugins/action_regex_project.py

@@ -9,7 +9,7 @@ from autocoder.common.code_auto_generate import CodeAutoGenerate
 from autocoder.common.code_auto_generate_diff import CodeAutoGenerateDiff
 from autocoder.common.code_auto_generate_strict_diff import CodeAutoGenerateStrictDiff
 from autocoder.common.code_auto_generate_editblock import CodeAutoGenerateEditBlock
-from autocoder.index.index import build_index_and_filter_files
+from autocoder.index.entry import build_index_and_filter_files
 from autocoder.regexproject import RegexProject
 from autocoder.utils.conversation_store import store_code_model_conversation
 from loguru import logger

autocoder/index/entry.py

@@ -0,0 +1,286 @@
+import os
+import json
+import time
+from typing import List, Dict, Any, Optional
+from datetime import datetime
+from autocoder.common import SourceCode, AutoCoderArgs
+
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+
+from loguru import logger
+from autocoder.utils.queue_communicate import (
+    queue_communicate,
+    CommunicateEvent,
+    CommunicateEventType,
+)
+from autocoder.index.types import (
+    TargetFile
+)
+
+from autocoder.index.filter.quick_filter import QuickFilter
+from autocoder.index.filter.normal_filter import NormalFilter
+from autocoder.index.index import IndexManager
+
+def build_index_and_filter_files(
+    llm, args: AutoCoderArgs, sources: List[SourceCode]
+) -> str:
+    # Initialize timing and statistics
+    total_start_time = time.monotonic()
+    stats = {
+        "total_files": len(sources),
+        "indexed_files": 0,
+        "level1_filtered": 0,
+        "level2_filtered": 0,
+        "verified_files": 0,
+        "final_files": 0,
+        "timings": {
+            "process_tagged_sources": 0.0,
+            "build_index": 0.0,
+            "quick_filter": 0.0,
+            "normal_filter": {
+                "level1_filter": 0.0,
+                "level2_filter": 0.0,
+                "relevance_verification": 0.0,
+            },
+            "file_selection": 0.0,
+            "prepare_output": 0.0,
+            "total": 0.0
+        }
+    }
+
+    def get_file_path(file_path):
+        if file_path.startswith("##"):
+            return file_path.strip()[2:]
+        return file_path
+
+    final_files: Dict[str, TargetFile] = {}
+
+    # Phase 1: Process REST/RAG/Search sources
+    logger.info("Phase 1: Processing REST/RAG/Search sources...")
+    phase_start = time.monotonic()
+    for source in sources:
+        if source.tag in ["REST", "RAG", "SEARCH"]:
+            final_files[get_file_path(source.module_name)] = TargetFile(
+                file_path=source.module_name, reason="Rest/Rag/Search"
+            )
+    phase_end = time.monotonic()
+    stats["timings"]["process_tagged_sources"] = phase_end - phase_start
+    
+    if not args.skip_build_index and llm:
+        # Phase 2: Build index
+        if args.request_id and not args.skip_events:
+            queue_communicate.send_event(
+                request_id=args.request_id,
+                event=CommunicateEvent(
+                    event_type=CommunicateEventType.CODE_INDEX_BUILD_START.value,
+                    data=json.dumps({"total_files": len(sources)})
+                )
+            )
+
+        logger.info("Phase 2: Building index for all files...")
+        phase_start = time.monotonic()
+        index_manager = IndexManager(llm=llm, sources=sources, args=args)
+        index_data = index_manager.build_index()
+        stats["indexed_files"] = len(index_data) if index_data else 0
+        phase_end = time.monotonic()
+        stats["timings"]["build_index"] = phase_end - phase_start
+
+        if args.request_id and not args.skip_events:
+            queue_communicate.send_event(
+                request_id=args.request_id,
+                event=CommunicateEvent(
+                    event_type=CommunicateEventType.CODE_INDEX_BUILD_END.value,
+                    data=json.dumps({
+                        "indexed_files": stats["indexed_files"],
+                        "build_index_time": stats["timings"]["build_index"],
+                    })
+                )
+            )
+
+        quick_filter = QuickFilter(index_manager,stats,sources)
+        final_files = quick_filter.filter(index_manager.read_index(),args.query)
+        
+        if not final_files:
+            normal_filter = NormalFilter(index_manager,stats,sources)
+            final_files = normal_filter.filter(index_manager.read_index(),args.query)
+        
+
+    def display_table_and_get_selections(data):
+        from prompt_toolkit.shortcuts import checkboxlist_dialog
+        from prompt_toolkit.styles import Style
+
+        choices = [(file, f"{file} - {reason}") for file, reason in data]
+        selected_files = [file for file, _ in choices]
+
+        style = Style.from_dict(
+            {
+                "dialog": "bg:#88ff88",
+                "dialog frame.label": "bg:#ffffff #000000",
+                "dialog.body": "bg:#88ff88 #000000",
+                "dialog shadow": "bg:#00aa00",
+            }
+        )
+
+        result = checkboxlist_dialog(
+            title="Target Files",
+            text="Tab to switch between buttons, and Space/Enter to select/deselect.",
+            values=choices,
+            style=style,
+            default_values=selected_files,
+        ).run()
+
+        return [file for file in result] if result else []
+
+    def print_selected(data):
+        console = Console()
+
+        table = Table(
+            title="Files Used as Context",
+            show_header=True,
+            header_style="bold magenta",
+        )
+        table.add_column("File Path", style="cyan", no_wrap=True)
+        table.add_column("Reason", style="green")
+
+        for file, reason in data:
+            table.add_row(file, reason)
+
+        panel = Panel(
+            table,
+            expand=False,
+            border_style="bold blue",
+            padding=(1, 1),
+        )
+
+        console.print(panel)
+
+    # Phase 6: File selection and limitation
+    logger.info("Phase 6: Processing file selection and limits...")
+    phase_start = time.monotonic()
+
+    if args.index_filter_file_num > 0:
+        logger.info(
+            f"Limiting files from {len(final_files)} to {args.index_filter_file_num}")
+
+    if args.skip_confirm:
+        final_filenames = [file.file_path for file in final_files.values()]
+        if args.index_filter_file_num > 0:
+            final_filenames = final_filenames[: args.index_filter_file_num]
+    else:
+        target_files_data = [
+            (file.file_path, file.reason) for file in final_files.values()
+        ]
+        if not target_files_data:
+            logger.warning(
+                "No target files found, you may need to rewrite the query and try again."
+            )
+            final_filenames = []
+        else:
+            final_filenames = display_table_and_get_selections(
+                target_files_data)
+
+        if args.index_filter_file_num > 0:
+            final_filenames = final_filenames[: args.index_filter_file_num]
+
+    phase_end = time.monotonic()
+    stats["timings"]["file_selection"] = phase_end - phase_start
+
+    # Phase 7: Display results and prepare output
+    logger.info("Phase 7: Preparing final output...")
+    phase_start = time.monotonic()    
+    try:
+        print_selected(
+            [
+                (file.file_path, file.reason)
+                for file in final_files.values()
+                if file.file_path in final_filenames
+            ]
+        )
+    except Exception as e:
+        logger.warning(
+            "Failed to display selected files in terminal mode. Falling back to simple print."
+        )
+        print("Target Files Selected:")
+        for file in final_filenames:
+            print(f"{file} - {final_files[file].reason}")
+
+    source_code = ""
+    depulicated_sources = set()
+
+    for file in sources:
+        if file.module_name in final_filenames:
+            if file.module_name in depulicated_sources:
+                continue
+            depulicated_sources.add(file.module_name)
+            source_code += f"##File: {file.module_name}\n"
+            source_code += f"{file.source_code}\n\n"
+
+    if args.request_id and not args.skip_events:
+        queue_communicate.send_event(
+            request_id=args.request_id,
+            event=CommunicateEvent(
+                event_type=CommunicateEventType.CODE_INDEX_FILTER_FILE_SELECTED.value,
+                data=json.dumps([
+                    (file.file_path, file.reason)
+                    for file in final_files.values()
+                    if file.file_path in depulicated_sources
+                ])
+            )
+        )        
+
+    stats["final_files"] = len(depulicated_sources)
+    phase_end = time.monotonic()
+    stats["timings"]["prepare_output"] = phase_end - phase_start
+
+    # Calculate total time and print summary
+    total_end_time = time.monotonic()
+    total_time = total_end_time - total_start_time
+    stats["timings"]["total"] = total_time
+    
+    # Calculate total filter time
+    total_filter_time = (
+        stats["timings"]["quick_filter"] +
+        stats["timings"]["normal_filter"]["level1_filter"] +
+        stats["timings"]["normal_filter"]["level2_filter"] +
+        stats["timings"]["normal_filter"]["relevance_verification"]
+    )
+
+    # Print final statistics in a more structured way
+    summary = f"""
+=== Indexing and Filtering Summary ===
+• Total files scanned: {stats['total_files']}
+• Files indexed: {stats['indexed_files']}
+• Files filtered:
+  - Level 1 (query-based): {stats['level1_filtered']}
+  - Level 2 (related files): {stats['level2_filtered']}
+  - Relevance verified: {stats.get('verified_files', 0)}
+• Final files selected: {stats['final_files']}
+
+=== Time Breakdown ===
+• Index build: {stats['timings'].get('build_index', 0):.2f}s
+• Quick filter: {stats['timings'].get('quick_filter', 0):.2f}s
+• Normal filter: 
+    - Level 1 filter: {stats['timings']["normal_filter"].get('level1_filter', 0):.2f}s
+    - Level 2 filter: {stats['timings']["normal_filter"].get('level2_filter', 0):.2f}s
+    - Relevance check: {stats['timings']["normal_filter"].get('relevance_verification', 0):.2f}s
+• File selection: {stats['timings'].get('file_selection', 0):.2f}s
+• Total time: {total_time:.2f}s
+====================================
+"""
+    logger.info(summary)
+
+    if args.request_id and not args.skip_events:
+        queue_communicate.send_event(
+            request_id=args.request_id,
+            event=CommunicateEvent(
+                event_type=CommunicateEventType.CODE_INDEX_FILTER_END.value,
+                data=json.dumps({
+                    "filtered_files": stats["final_files"],
+                    "filter_time": total_filter_time
+                })
+            )
+        )
+
+    return source_code

autocoder/index/filter/normal_filter.py

@@ -0,0 +1,168 @@
+from typing import List, Union,Dict,Any
+from autocoder.index.types import IndexItem
+from autocoder.common import SourceCode, AutoCoderArgs
+import byzerllm
+import time
+from autocoder.index.index import IndexManager
+from autocoder.index.types import (
+    IndexItem,
+    TargetFile,
+    VerifyFileRelevance,
+    FileList,
+    FileNumberList
+)
+from loguru import logger
+from autocoder.utils.queue_communicate import (
+    queue_communicate,
+    CommunicateEvent,
+    CommunicateEventType,
+)
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import json
+
+def get_file_path(file_path):
+    if file_path.startswith("##"):
+        return file_path.strip()[2:]
+    return file_path
+
+
+class NormalFilter():
+    def __init__(self, index_manager: IndexManager,stats:Dict[str,Any],sources:List[SourceCode]):
+        self.index_manager = index_manager
+        self.args = index_manager.args
+        self.stats = stats
+        self.sources = sources
+
+    def filter(self, index_items: List[IndexItem], query: str) -> Dict[str, TargetFile]:
+        final_files: Dict[str, TargetFile] = {}
+        if not self.args.skip_filter_index:
+            if self.args.request_id and not self.args.skip_events:
+                queue_communicate.send_event(
+                    request_id=self.args.request_id,
+                    event=CommunicateEvent(
+                        event_type=CommunicateEventType.CODE_INDEX_FILTER_START.value,
+                        data=json.dumps({})
+                    )
+                )
+            # Phase 3: Level 1 filtering - Query-based
+            logger.info(
+                "Phase 3: Performing Level 1 filtering (query-based)...")
+
+            phase_start = time.monotonic()
+            target_files = self.index_manager.get_target_files_by_query(self.args.query)
+
+            if target_files:
+                for file in target_files.file_list:
+                    file_path = file.file_path.strip()
+                    final_files[get_file_path(file_path)] = file
+                self.stats["level1_filtered"] = len(target_files.file_list)
+            phase_end = time.monotonic()
+            self.stats["timings"]["normal_filter"]["level1_filter"] = phase_end - phase_start
+
+            # Phase 4: Level 2 filtering - Related files
+            if target_files is not None and self.args.index_filter_level >= 2:
+                logger.info(
+                    "Phase 4: Performing Level 2 filtering (related files)...")
+                if self.args.request_id and not self.args.skip_events:
+                    queue_communicate.send_event(
+                        request_id=self.args.request_id,
+                        event=CommunicateEvent(
+                            event_type=CommunicateEventType.CODE_INDEX_FILTER_START.value,
+                            data=json.dumps({})
+                        )
+                    )
+                phase_start = time.monotonic()
+                related_files = self.index_manager.get_related_files(
+                    [file.file_path for file in target_files.file_list]
+                )
+                if related_files is not None:
+                    for file in related_files.file_list:
+                        file_path = file.file_path.strip()
+                        final_files[get_file_path(file_path)] = file
+                    self.stats["level2_filtered"] = len(related_files.file_list)
+                phase_end = time.monotonic()
+                self.stats["timings"]["normal_filter"]["level2_filter"] = phase_end - phase_start
+
+            if not final_files:
+                logger.warning("No related files found, using all files")
+                for source in self.sources:
+                    final_files[get_file_path(source.module_name)] = TargetFile(
+                        file_path=source.module_name,
+                        reason="No related files found, use all files",
+                    )
+
+            # Phase 5: Relevance verification
+            logger.info("Phase 5: Performing relevance verification...")
+            if self.args.index_filter_enable_relevance_verification:
+                phase_start = time.monotonic()
+                verified_files = {}
+                temp_files = list(final_files.values())
+                verification_results = []
+                
+                def print_verification_results(results):
+                    from rich.table import Table
+                    from rich.console import Console
+                    
+                    console = Console()
+                    table = Table(title="File Relevance Verification Results", show_header=True, header_style="bold magenta")
+                    table.add_column("File Path", style="cyan", no_wrap=True)
+                    table.add_column("Score", justify="right", style="green")
+                    table.add_column("Status", style="yellow")
+                    table.add_column("Reason/Error")
+                    
+                    for file_path, score, status, reason in results:
+                        table.add_row(
+                            file_path,
+                            str(score) if score is not None else "N/A",
+                            status,
+                            reason
+                        )
+                    
+                    console.print(table)
+
+                def verify_single_file(file: TargetFile):
+                    for source in self.sources:
+                        if source.module_name == file.file_path:
+                            file_content = source.source_code
+                            try:
+                                result = self.index_manager.verify_file_relevance.with_llm(llm).with_return_type(VerifyFileRelevance).run(
+                                    file_content=file_content,
+                                    query=self.args.query
+                                )
+                                if result.relevant_score >= self.args.verify_file_relevance_score:
+                                    verified_files[file.file_path] = TargetFile(
+                                        file_path=file.file_path,
+                                        reason=f"Score:{result.relevant_score}, {result.reason}"
+                                    )
+                                    return file.file_path, result.relevant_score, "PASS", result.reason
+                                else:
+                                    return file.file_path, result.relevant_score, "FAIL", result.reason
+                            except Exception as e:
+                                error_msg = str(e)
+                                verified_files[file.file_path] = TargetFile(
+                                    file_path=file.file_path,
+                                    reason=f"Verification failed: {error_msg}"
+                                )
+                                return file.file_path, None, "ERROR", error_msg
+                    return None
+
+                with ThreadPoolExecutor(max_workers=self.args.index_filter_workers) as executor:
+                    futures = [executor.submit(verify_single_file, file)
+                            for file in temp_files]
+                    for future in as_completed(futures):
+                        result = future.result()
+                        if result:
+                            verification_results.append(result)
+                            time.sleep(self.args.anti_quota_limit)
+
+                # Print verification results in a table
+                print_verification_results(verification_results)
+                
+                self.stats["verified_files"] = len(verified_files)
+                phase_end = time.monotonic()
+                self.stats["timings"]["normal_filter"]["relevance_verification"] = phase_end - phase_start
+
+                # Keep all files, not just verified ones
+                final_files = verified_files
+        
+        return final_files

autocoder/index/filter/quick_filter.py

@@ -0,0 +1,98 @@
+from typing import List, Union,Dict,Any
+from autocoder.index.types import IndexItem
+from autocoder.common import AutoCoderArgs,SourceCode
+import byzerllm
+import time
+from autocoder.index.index import IndexManager
+from autocoder.index.types import (
+    IndexItem,
+    TargetFile,    
+    FileNumberList
+)
+from autocoder.rag.token_counter import count_tokens
+from loguru import logger
+
+
+def get_file_path(file_path):
+    if file_path.startswith("##"):
+        return file_path.strip()[2:]
+    return file_path
+
+
+class QuickFilter():
+    def __init__(self, index_manager: IndexManager,stats:Dict[str,Any],sources:List[SourceCode]):
+        self.index_manager = index_manager
+        self.args = index_manager.args
+        self.stats = stats
+        self.sources = sources
+
+    @byzerllm.prompt()
+    def quick_filter_files(self,file_meta_list:List[IndexItem],query:str) -> str:
+        '''
+        当用户提一个需求的时候，我们需要找到相关的文件，然后阅读这些文件，并且修改其中部分文件。
+        现在，给定下面的索引文件：
+
+        <index>
+        {{ content }}
+        </index>
+
+        索引文件包含文件序号(##[]括起来的部分)，文件路径，文件符号信息等。
+        下面是用户的查询需求：
+        
+        <query>
+        {{ query }}
+        </query>
+
+        请根据用户的需求，找到相关的文件，并给出文件序号列表。请返回如下json格式：
+
+        ```json
+        {
+            "file_list": [
+                file_index1,
+                file_index2,
+                ...
+            ]
+        }
+        ```
+
+        特别注意,如果用户的query里 @文件 或者 @@符号，那么被@的文件或者@@的符号必须要返回，并且查看他们依赖的文件是否相关。        
+        '''
+        file_meta_str = "\n".join([f"##[{index}]{item.module_name}\n{item.symbols}" for index,item in enumerate(file_meta_list)])
+        context = {
+            "content": file_meta_str,
+            "query": query
+        }
+        return context    
+
+    def filter(self, index_items: List[IndexItem], query: str) -> Dict[str, TargetFile]:
+        final_files: Dict[str, TargetFile] = {}
+        if not self.args.skip_filter_index and self.args.index_filter_model:
+            start_time = time.monotonic()
+            index_items = self.index_manager.read_index()
+
+            prompt_str = self.quick_filter_files.prompt(index_items,query)
+
+            print(prompt_str)
+            
+            tokens_len = count_tokens(prompt_str)
+            
+            if tokens_len > 55*1024:
+                logger.warning(f"Quick filter prompt is too long, tokens_len: {tokens_len}/{55*1024} fallback to normal filter")
+                return final_files
+            
+            try:
+                file_number_list = self.quick_filter_files.with_llm(
+                    self.index_manager.index_filter_llm).with_return_type(FileNumberList).run(index_items, self.args.query)
+            except Exception as e:
+                logger.error(f"Quick filter failed, error: {str(e)} fallback to normal filter")
+                return final_files
+            
+            if file_number_list:
+                for file_number in file_number_list.file_list:
+                    final_files[get_file_path(index_items[file_number].module_name)] = TargetFile(
+                        file_path=index_items[file_number].module_name,
+                        reason="Quick Filter"
+                    )
+            end_time = time.monotonic()            
+            self.stats["timings"]["quick_filter"] = end_time - start_time            
+        return final_files

autocoder/index/index.py

@@ -6,37 +6,22 @@ from datetime import datetime
 from autocoder.common import SourceCode, AutoCoderArgs
 from autocoder.index.symbols_utils import (
     extract_symbols,
-    SymbolsInfo,
     SymbolType,
     symbols_info_to_str,
 )
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import threading
 
-import pydantic
 import byzerllm
 import hashlib
-import textwrap
-import tabulate
-from rich.console import Console
-from rich.table import Table
-from rich.panel import Panel
-from rich.text import Text
 
 from loguru import logger
-from autocoder.utils.queue_communicate import (
-    queue_communicate,
-    CommunicateEvent,
-    CommunicateEventType,
-)
 from autocoder.index.types import (
     IndexItem,
     TargetFile,
-    VerifyFileRelevance,
     FileList,
 )
 
-
 class IndexManager:
     def __init__(
         self, llm: byzerllm.ByzerLLM, sources: List[SourceCode], args: AutoCoderArgs
@@ -53,6 +38,11 @@ class IndexManager:
         else:
             self.index_llm = llm
 
+        if llm and (s := llm.get_sub_client("index_filter_model")):
+            self.index_filter_llm = s
+        else:
+            self.index_filter_llm = llm
+
         self.llm = llm
         self.args = args
         self.max_input_length = (
@@ -63,6 +53,7 @@ class IndexManager:
         if not os.path.exists(self.index_dir):
             os.makedirs(self.index_dir)
 
+    
     @byzerllm.prompt()
     def verify_file_relevance(self, file_content: str, query: str) -> str:
         """
@@ -565,381 +556,3 @@ class IndexManager:
         请确保结果的准确性和完整性，包括所有可能相关的文件。
         """
 
-
-def build_index_and_filter_files(
-    llm, args: AutoCoderArgs, sources: List[SourceCode]
-) -> str:
-    # Initialize timing and statistics
-    total_start_time = time.monotonic()
-    stats = {
-        "total_files": len(sources),
-        "indexed_files": 0,
-        "level1_filtered": 0,
-        "level2_filtered": 0,
-        "verified_files": 0,
-        "final_files": 0,
-        "timings": {
-            "process_tagged_sources": 0.0,
-            "build_index": 0.0,
-            "level1_filter": 0.0,
-            "level2_filter": 0.0,
-            "relevance_verification": 0.0,
-            "file_selection": 0.0,
-            "prepare_output": 0.0,
-            "total": 0.0
-        }
-    }
-
-    def get_file_path(file_path):
-        if file_path.startswith("##"):
-            return file_path.strip()[2:]
-        return file_path
-
-    final_files: Dict[str, TargetFile] = {}
-
-    # Phase 1: Process REST/RAG/Search sources
-    logger.info("Phase 1: Processing REST/RAG/Search sources...")
-    phase_start = time.monotonic()
-    for source in sources:
-        if source.tag in ["REST", "RAG", "SEARCH"]:
-            final_files[get_file_path(source.module_name)] = TargetFile(
-                file_path=source.module_name, reason="Rest/Rag/Search"
-            )
-    phase_end = time.monotonic()
-    stats["timings"]["process_tagged_sources"] = phase_end - phase_start
-
-    if not args.skip_build_index and llm:
-        # Phase 2: Build index
-        if args.request_id and not args.skip_events:
-            queue_communicate.send_event(
-                request_id=args.request_id,
-                event=CommunicateEvent(
-                    event_type=CommunicateEventType.CODE_INDEX_BUILD_START.value,
-                    data=json.dumps({"total_files": len(sources)})
-                )
-            )
-
-        logger.info("Phase 2: Building index for all files...")
-        phase_start = time.monotonic()
-        index_manager = IndexManager(llm=llm, sources=sources, args=args)
-        index_data = index_manager.build_index()
-        stats["indexed_files"] = len(index_data) if index_data else 0
-        phase_end = time.monotonic()
-        stats["timings"]["build_index"] = phase_end - phase_start
-
-        if args.request_id and not args.skip_events:
-            queue_communicate.send_event(
-                request_id=args.request_id,
-                event=CommunicateEvent(
-                    event_type=CommunicateEventType.CODE_INDEX_BUILD_END.value,
-                    data=json.dumps({
-                        "indexed_files": stats["indexed_files"],
-                        "build_index_time": stats["timings"]["build_index"],
-                    })
-                )
-            )
-
-        if not args.skip_filter_index:
-            if args.request_id and not args.skip_events:
-                queue_communicate.send_event(
-                    request_id=args.request_id,
-                    event=CommunicateEvent(
-                        event_type=CommunicateEventType.CODE_INDEX_FILTER_START.value,
-                        data=json.dumps({})
-                    )
-                )
-            # Phase 3: Level 1 filtering - Query-based
-            logger.info(
-                "Phase 3: Performing Level 1 filtering (query-based)...")
-
-            phase_start = time.monotonic()
-            target_files = index_manager.get_target_files_by_query(args.query)
-
-            if target_files:
-                for file in target_files.file_list:
-                    file_path = file.file_path.strip()
-                    final_files[get_file_path(file_path)] = file
-                stats["level1_filtered"] = len(target_files.file_list)
-            phase_end = time.monotonic()
-            stats["timings"]["level1_filter"] = phase_end - phase_start
-
-            # Phase 4: Level 2 filtering - Related files
-            if target_files is not None and args.index_filter_level >= 2:
-                logger.info(
-                    "Phase 4: Performing Level 2 filtering (related files)...")
-                if args.request_id and not args.skip_events:
-                    queue_communicate.send_event(
-                        request_id=args.request_id,
-                        event=CommunicateEvent(
-                            event_type=CommunicateEventType.CODE_INDEX_FILTER_START.value,
-                            data=json.dumps({})
-                        )
-                    )
-                phase_start = time.monotonic()
-                related_files = index_manager.get_related_files(
-                    [file.file_path for file in target_files.file_list]
-                )
-                if related_files is not None:
-                    for file in related_files.file_list:
-                        file_path = file.file_path.strip()
-                        final_files[get_file_path(file_path)] = file
-                    stats["level2_filtered"] = len(related_files.file_list)
-                phase_end = time.monotonic()
-                stats["timings"]["level2_filter"] = phase_end - phase_start
-
-            if not final_files:
-                logger.warning("No related files found, using all files")
-                for source in sources:
-                    final_files[get_file_path(source.module_name)] = TargetFile(
-                        file_path=source.module_name,
-                        reason="No related files found, use all files",
-                    )
-
-            # Phase 5: Relevance verification
-            logger.info("Phase 5: Performing relevance verification...")
-            if args.index_filter_enable_relevance_verification:
-                phase_start = time.monotonic()
-                verified_files = {}
-                temp_files = list(final_files.values())
-                verification_results = []
-                
-                def print_verification_results(results):
-                    from rich.table import Table
-                    from rich.console import Console
-                    
-                    console = Console()
-                    table = Table(title="File Relevance Verification Results", show_header=True, header_style="bold magenta")
-                    table.add_column("File Path", style="cyan", no_wrap=True)
-                    table.add_column("Score", justify="right", style="green")
-                    table.add_column("Status", style="yellow")
-                    table.add_column("Reason/Error")
-                    
-                    for file_path, score, status, reason in results:
-                        table.add_row(
-                            file_path,
-                            str(score) if score is not None else "N/A",
-                            status,
-                            reason
-                        )
-                    
-                    console.print(table)
-
-                def verify_single_file(file: TargetFile):
-                    for source in sources:
-                        if source.module_name == file.file_path:
-                            file_content = source.source_code
-                            try:
-                                result = index_manager.verify_file_relevance.with_llm(llm).with_return_type(VerifyFileRelevance).run(
-                                    file_content=file_content,
-                                    query=args.query
-                                )
-                                if result.relevant_score >= args.verify_file_relevance_score:
-                                    verified_files[file.file_path] = TargetFile(
-                                        file_path=file.file_path,
-                                        reason=f"Score:{result.relevant_score}, {result.reason}"
-                                    )
-                                    return file.file_path, result.relevant_score, "PASS", result.reason
-                                else:
-                                    return file.file_path, result.relevant_score, "FAIL", result.reason
-                            except Exception as e:
-                                error_msg = str(e)
-                                verified_files[file.file_path] = TargetFile(
-                                    file_path=file.file_path,
-                                    reason=f"Verification failed: {error_msg}"
-                                )
-                                return file.file_path, None, "ERROR", error_msg
-                    return None
-
-                with ThreadPoolExecutor(max_workers=args.index_filter_workers) as executor:
-                    futures = [executor.submit(verify_single_file, file)
-                            for file in temp_files]
-                    for future in as_completed(futures):
-                        result = future.result()
-                        if result:
-                            verification_results.append(result)
-                            time.sleep(args.anti_quota_limit)
-
-                # Print verification results in a table
-                print_verification_results(verification_results)
-                
-                stats["verified_files"] = len(verified_files)
-                phase_end = time.monotonic()
-                stats["timings"]["relevance_verification"] = phase_end - phase_start
-
-                # Keep all files, not just verified ones
-                final_files = verified_files
-
-    def display_table_and_get_selections(data):
-        from prompt_toolkit.shortcuts import checkboxlist_dialog
-        from prompt_toolkit.styles import Style
-
-        choices = [(file, f"{file} - {reason}") for file, reason in data]
-        selected_files = [file for file, _ in choices]
-
-        style = Style.from_dict(
-            {
-                "dialog": "bg:#88ff88",
-                "dialog frame.label": "bg:#ffffff #000000",
-                "dialog.body": "bg:#88ff88 #000000",
-                "dialog shadow": "bg:#00aa00",
-            }
-        )
-
-        result = checkboxlist_dialog(
-            title="Target Files",
-            text="Tab to switch between buttons, and Space/Enter to select/deselect.",
-            values=choices,
-            style=style,
-            default_values=selected_files,
-        ).run()
-
-        return [file for file in result] if result else []
-
-    def print_selected(data):
-        console = Console()
-
-        table = Table(
-            title="Files Used as Context",
-            show_header=True,
-            header_style="bold magenta",
-        )
-        table.add_column("File Path", style="cyan", no_wrap=True)
-        table.add_column("Reason", style="green")
-
-        for file, reason in data:
-            table.add_row(file, reason)
-
-        panel = Panel(
-            table,
-            expand=False,
-            border_style="bold blue",
-            padding=(1, 1),
-        )
-
-        console.print(panel)
-
-    # Phase 6: File selection and limitation
-    logger.info("Phase 6: Processing file selection and limits...")
-    phase_start = time.monotonic()
-
-    if args.index_filter_file_num > 0:
-        logger.info(
-            f"Limiting files from {len(final_files)} to {args.index_filter_file_num}")
-
-    if args.skip_confirm:
-        final_filenames = [file.file_path for file in final_files.values()]
-        if args.index_filter_file_num > 0:
-            final_filenames = final_filenames[: args.index_filter_file_num]
-    else:
-        target_files_data = [
-            (file.file_path, file.reason) for file in final_files.values()
-        ]
-        if not target_files_data:
-            logger.warning(
-                "No target files found, you may need to rewrite the query and try again."
-            )
-            final_filenames = []
-        else:
-            final_filenames = display_table_and_get_selections(
-                target_files_data)
-
-        if args.index_filter_file_num > 0:
-            final_filenames = final_filenames[: args.index_filter_file_num]
-
-    phase_end = time.monotonic()
-    stats["timings"]["file_selection"] = phase_end - phase_start
-
-    # Phase 7: Display results and prepare output
-    logger.info("Phase 7: Preparing final output...")
-    phase_start = time.monotonic()    
-    try:
-        print_selected(
-            [
-                (file.file_path, file.reason)
-                for file in final_files.values()
-                if file.file_path in final_filenames
-            ]
-        )
-    except Exception as e:
-        logger.warning(
-            "Failed to display selected files in terminal mode. Falling back to simple print."
-        )
-        print("Target Files Selected:")
-        for file in final_filenames:
-            print(f"{file} - {final_files[file].reason}")
-
-    source_code = ""
-    depulicated_sources = set()
-
-    for file in sources:
-        if file.module_name in final_filenames:
-            if file.module_name in depulicated_sources:
-                continue
-            depulicated_sources.add(file.module_name)
-            source_code += f"##File: {file.module_name}\n"
-            source_code += f"{file.source_code}\n\n"
-
-    if args.request_id and not args.skip_events:
-        queue_communicate.send_event(
-            request_id=args.request_id,
-            event=CommunicateEvent(
-                event_type=CommunicateEventType.CODE_INDEX_FILTER_FILE_SELECTED.value,
-                data=json.dumps([
-                    (file.file_path, file.reason)
-                    for file in final_files.values()
-                    if file.file_path in depulicated_sources
-                ])
-            )
-        )        
-
-    stats["final_files"] = len(depulicated_sources)
-    phase_end = time.monotonic()
-    stats["timings"]["prepare_output"] = phase_end - phase_start
-
-    # Calculate total time and print summary
-    total_end_time = time.monotonic()
-    total_time = total_end_time - total_start_time
-    stats["timings"]["total"] = total_time
-    
-    # Calculate total filter time
-    total_filter_time = (
-        stats["timings"]["level1_filter"] +
-        stats["timings"]["level2_filter"] +
-        stats["timings"]["relevance_verification"]
-    )
-
-    # Print final statistics in a more structured way
-    summary = f"""
-=== Indexing and Filtering Summary ===
-• Total files scanned: {stats['total_files']}
-• Files indexed: {stats['indexed_files']}
-• Files filtered:
-  - Level 1 (query-based): {stats['level1_filtered']}
-  - Level 2 (related files): {stats['level2_filtered']}
-  - Relevance verified: {stats.get('verified_files', 0)}
-• Final files selected: {stats['final_files']}
-
-=== Time Breakdown ===
-• Index build: {stats['timings'].get('build_index', 0):.2f}s
-• Level 1 filter: {stats['timings'].get('level1_filter', 0):.2f}s
-• Level 2 filter: {stats['timings'].get('level2_filter', 0):.2f}s
-• Relevance check: {stats['timings'].get('relevance_verification', 0):.2f}s
-• File selection: {stats['timings'].get('file_selection', 0):.2f}s
-• Total time: {total_time:.2f}s
-====================================
-"""
-    logger.info(summary)
-
-    if args.request_id and not args.skip_events:
-        queue_communicate.send_event(
-            request_id=args.request_id,
-            event=CommunicateEvent(
-                event_type=CommunicateEventType.CODE_INDEX_FILTER_END.value,
-                data=json.dumps({
-                    "filtered_files": stats["final_files"],
-                    "filter_time": total_filter_time
-                })
-            )
-        )
-
-    return source_code

autocoder/index/types.py

@@ -21,4 +21,7 @@ class VerifyFileRelevance(pydantic.BaseModel):
 
 
 class FileList(pydantic.BaseModel):
-    file_list: List[TargetFile]
+    file_list: List[TargetFile]
+
+class FileNumberList(pydantic.BaseModel):
+    file_list: List[int]    

autocoder/version.py

@@ -1 +1 @@
-__version__ = "0.1.232"
+__version__ = "0.1.233"