auto-coder 0.1.243__py3-none-any.whl → 0.1.245__py3-none-any.whl

autocoder/index/filter/quick_filter.py

@@ -69,7 +69,7 @@ class QuickFilter():
 
     def filter(self, index_items: List[IndexItem], query: str) -> Dict[str, TargetFile]:
         final_files: Dict[str, TargetFile] = {}
-        if not self.args.skip_filter_index and self.index_manager.llm.get_sub_client("index_filter_model"):
+        if not self.args.skip_filter_index and self.args.index_filter_model:
             start_time = time.monotonic()
             index_items = self.index_manager.read_index()
 

autocoder/index/index.py

@@ -130,6 +130,7 @@ class IndexManager:
         如果有符号，按如下格式返回:
 
         ```
+        用途：主要用于提供自动实现函数模板的功能。
         {符号类型}: {符号名称}, {符号名称}, ...
         ```
 

autocoder/models.py

@@ -2,6 +2,7 @@ import os
 import json
 from typing import List, Dict
 from urllib.parse import urlparse
+from autocoder.common.auto_coder_lang import get_message_with_format
 
 MODELS_JSON = os.path.expanduser("~/.auto-coder/keys/models.json")
 
@@ -36,6 +37,22 @@ default_models_list = [
     }
 ]
 
+def process_api_key_path(base_url: str) -> str:
+    """
+    从 base_url 中提取 host 部分并处理特殊字符
+    例如: https://api.example.com:8080/v1 -> api.example.com_8080
+    """
+    if not base_url:
+        return ""
+    
+    parsed = urlparse(base_url)
+    host = parsed.netloc
+    
+    # 将冒号替换为下划线
+    host = host.replace(":", "_")
+    
+    return host
+
 def load_models() -> List[Dict]:
     """
     Load models from ~/.auto-coder/keys/models.json and merge with default_models_list.
@@ -73,7 +90,7 @@ def load_models() -> List[Dict]:
             api_key_file = os.path.join(api_key_dir, model["api_key_path"])
             if os.path.exists(api_key_file):
                 with open(api_key_file, "r") as f:
-                    model["api_key"] = f.read()
+                    model["api_key"] = f.read()                   
     return target_models
 
 def save_models(models: List[Dict]) -> None:
@@ -85,22 +102,6 @@ def save_models(models: List[Dict]) -> None:
         json.dump(models, f, indent=2, ensure_ascii=False)
 
 
-def process_api_key_path(base_url: str) -> str:
-    """
-    从 base_url 中提取 host 部分并处理特殊字符
-    例如: https://api.example.com:8080/v1 -> api.example.com_8080
-    """
-    if not base_url:
-        return ""
-    
-    parsed = urlparse(base_url)
-    host = parsed.netloc
-    
-    # 将冒号替换为下划线
-    host = host.replace(":", "_")
-    
-    return host
-
 def get_model_by_name(name: str) -> Dict:
     """
     根据模型名称查找模型
@@ -108,8 +109,8 @@ def get_model_by_name(name: str) -> Dict:
     models = load_models()
     v = [m for m in models if m["name"] == name.strip()]
     
-    if len(v) == 0:
-        raise Exception(f"Model {name} not found")
+    if len(v) == 0:        
+        raise Exception(get_message_with_format("model_not_found", model_name=name))
     return v[0]
 
 def update_model_with_api_key(name: str, api_key: str) -> Dict:
@@ -135,9 +136,8 @@ def update_model_with_api_key(name: str, api_key: str) -> Dict:
                     
     if not found_model:
         return None
-        
-    # 从 base_url 中提取并处理 host
-    api_key_path = process_api_key_path(found_model["base_url"])
+            
+    api_key_path = name
     if api_key_path:
         found_model["api_key_path"] = api_key_path
         

autocoder/suffixproject/__init__.py

@@ -36,9 +36,8 @@ class SuffixProject:
         self.target_file = args.target_file
         self.project_type = args.project_type
         self.suffixs = [
-            f".{suffix.strip()}" if not suffix.startswith(".") else suffix.strip()
-            for suffix in self.project_type.split(",")
-            if suffix.strip() != ""
+            suffix.strip() if suffix.startswith(".") else f".{suffix.strip()}"
+            for suffix in self.project_type.split(",") if suffix.strip()
         ]
         self.file_filter = file_filter
         self.sources = []

autocoder/utils/auto_coder_utils/chat_stream_out.py

@@ -11,256 +11,132 @@ from autocoder.utils.request_queue import request_queue
 import time
 
 MAX_HISTORY_LINES = 40  # 最大保留历史行数
-LAYOUT_TYPES = Literal["vertical", "horizontal"]
 
-class StreamController:
-    def __init__(self, layout_type: LAYOUT_TYPES = "vertical", console: Optional[Console] = None):
-        self.console = console or Console(force_terminal=True, color_system="auto", height=24)  # 设置默认高度
-        self.layout = Layout()
-        self.queue = Queue()
+class StreamRenderer:
+    def __init__(self, title: str):
+        self.title = title
+        self.content = ""
         self.lock = Lock()
-        self.running = True
-        self.workers = []
-        self.layout_type = layout_type
-        self.stream_count = 0
-
-    def _create_stream_panel(self, idx: int) -> Layout:
-        """创建流面板布局"""
-        # 计算安全高度
-        current_height = self.console.height or 24  # 默认24行防止获取失败
-        safe_height = max(min(50, current_height // 2 - 4), 5)  # 限制最小高度为5行
-        
-        # 使用整数设置 Layout 的 size
-        panel = Layout(name=f"stream-{idx}", size=safe_height)
-        
-        panel.update(
-            Panel(
-                Markdown(""),
-                title=f"Stream {idx + 1}",
-                border_style="green",
-                height=safe_height  # 确保数值有效
-            )
-        )
-        return panel
-
-    def prepare_layout(self, count: int):
-        """准备动态布局结构"""
-        self.stream_count = count
-        
-        # 创建一个主布局容器
-        streams_layout = Layout(name="streams")
-        
-        # 创建所有流的布局
-        stream_layouts = []
-        for i in range(count):
-            stream_layout = Layout(name=f"stream-{i}")
-            panel = self._create_stream_panel(i)
-            stream_layout.update(panel)
-            stream_layouts.append(stream_layout)
-        
-        # 将所有流添加到主布局
-        if stream_layouts:
-            streams_layout.update(stream_layouts[0])
-            for i in range(1, len(stream_layouts)):
-                if self.layout_type == "vertical":
-                    streams_layout.split_column(stream_layouts[i])
-                elif self.layout_type == "horizontal":
-                    streams_layout.split_row(stream_layouts[i])
-                else:
-                    streams_layout.split_column(stream_layouts[i])
+        self.is_complete = False
         
-        # header 与 streams 布局分开
-        self.layout.split(
-            Layout(name="header", size=1),
-            streams_layout
-        )
-
-    def update_panel(self, idx: int, content: str, final: bool = False):
-        """线程安全的面板更新方法"""
+    def update(self, content: str):
         with self.lock:
-            # 计算安全高度
-            safe_height = min(50, self.console.height // 2 - 4)
+            self.content += content
             
-            if final:
-                new_panel = Panel(
-                    Markdown(content),
-                    title=f"Final Stream {idx+1}",
-                    border_style="blue",
-                    height=safe_height
-                )
-            else:
-                new_panel = Panel(
-                    Markdown(content),
-                    title=f"Stream {idx+1}",
-                    border_style="green",
-                    height=safe_height
-                )
+    def get_content(self) -> str:
+        with self.lock:
+            return self.content
+            
+    def complete(self):
+        with self.lock:
+            self.is_complete = True
 
-            panel_name = f"stream-{idx}"
-            streams_layout = self.layout["streams"]
+class MultiStreamRenderer:
+    def __init__(self, stream_titles: List[str], layout: str = "horizontal", console: Optional[Console] = None):
+        """
+        Initialize multi-stream renderer
+        
+        Args:
+            stream_titles: List of titles for each stream
+            layout: "horizontal" or "vertical"
+            console: Rich console instance
+        """
+        if console is None:
+            console = Console(force_terminal=True, color_system="auto")
             
-            # 递归查找目标布局
-            def find_layout(layout, name):
-                if layout.name == name:
-                    return layout
-                for child in layout.children:
-                    result = find_layout(child, name)
-                    if result:
-                        return result
-                return None
+        self.console = console
+        self.layout_type = layout
+        self.streams = [StreamRenderer(title) for title in stream_titles]
+        self.layout = Layout()
+        
+        # Create named layouts for each stream
+        self.stream_layouts = [Layout(name=f"stream{i}") for i in range(len(stream_titles))]
+        
+        # Configure layout
+        if layout == "horizontal":
+            self.layout.split_row(*self.stream_layouts)
+        else:
+            self.layout.split_column(*self.stream_layouts)
             
-            # 查找并更新目标布局
-            target_layout = find_layout(streams_layout, panel_name)
-            if target_layout:
-                target_layout.update(new_panel)
-            else:
-                import logging
-                logging.warning(f"未找到布局 {panel_name}，无法更新面板。")
+    def _process_stream(self, 
+                       stream_idx: int, 
+                       stream_generator: Generator[Tuple[str, Dict[str, Any]], None, None]):
+        """Process a single stream in a separate thread"""
+        stream = self.streams[stream_idx]
+        try:
+            for content, meta in stream_generator:
+                if content:
+                    stream.update(content)
+        finally:
+            stream.complete()
 
-def stream_worker(
-    idx: int,
-    generator: Generator[Tuple[str, Dict[str, Any]], None, None],
-    controller: StreamController,
-    request_id: Optional[str] = None
-) -> Tuple[str, Optional[Dict[str, Any]]]:
-    """单个流处理工作线程"""
-    lines_buffer = []
-    current_line = ""
-    assistant_response = ""
-    last_meta = None
-    
-    try:
-        for res in generator:
-            content, meta = res
-            last_meta = meta
+    def render_streams(self, 
+                      stream_generators: List[Generator[Tuple[str, Dict[str, Any]], None, None]]) -> List[str]:
+        """
+        Render multiple streams simultaneously
+        
+        Args:
+            stream_generators: List of stream generators to render
             
-            assistant_response += content
-            display_delta = meta.reasoning_content or content
-
-            parts = (current_line + display_delta).split("\n")
-            if len(parts) > 1:
-                lines_buffer.extend(parts[:-1])
-                if len(lines_buffer) > MAX_HISTORY_LINES:
-                    del lines_buffer[0:len(lines_buffer) - MAX_HISTORY_LINES]
+        Returns:
+            List of final content from each stream
+        """
+        assert len(stream_generators) == len(self.streams), "Number of generators must match number of streams"
+        
+        # Start processing threads
+        threads = []
+        for i, generator in enumerate(stream_generators):
+            thread = Thread(target=self._process_stream, args=(i, generator))
+            thread.daemon = True
+            thread.start()
+            threads.append(thread)
             
-            current_line = parts[-1]
-            display_content = "\n".join(lines_buffer[-MAX_HISTORY_LINES:] + [current_line])
+        try:
+            with Live(self.layout, console=self.console, refresh_per_second=10) as live:
+                while any(not stream.is_complete for stream in self.streams):
+                    # Update all panels
+                    for i, stream in enumerate(self.streams):
+                        panel = Panel(
+                            Markdown(stream.get_content() or "Waiting..."),
+                            title=stream.title,
+                            border_style="green" if not stream.is_complete else "blue"
+                        )
+                        
+                        # Update appropriate layout section
+                        self.stream_layouts[i].update(panel)
+                        
+                    time.sleep(0.1)  # Prevent excessive CPU usage
+                    
+        except KeyboardInterrupt:
+            print("\nStopping streams...")
             
-            controller.queue.put((idx, display_content, False))
+        # Wait for all threads to complete
+        for thread in threads:
+            thread.join()
             
-            if request_id and request_queue:
-                request_queue.add_request(
-                    request_id,
-                    RequestValue(
-                        value=StreamValue(value=[content]),
-                        status=RequestOption.RUNNING,
-                    ),
-                )
-
-        if current_line:
-            lines_buffer.append(current_line)
-        controller.queue.put((idx, assistant_response, True))
-        return assistant_response, last_meta
-    
-    except Exception as e:
-        error_content = f"Error: {str(e)}"
-        controller.queue.put((idx, error_content, True))
-        if request_id and request_queue:
-            request_queue.add_request(
-                request_id,
-                RequestValue(
-                    value=StreamValue(value=[str(e)]), 
-                    status=RequestOption.FAILED
-                ),
-            )
-        return assistant_response, last_meta
-    finally:
-        if request_id and request_queue:
-            request_queue.add_request(
-                request_id,
-                RequestValue(
-                    value=StreamValue(value=[""]), 
-                    status=RequestOption.COMPLETED
-                ),
-            )
+        return [stream.get_content() for stream in self.streams]
 
 def multi_stream_out(
     stream_generators: List[Generator[Tuple[str, Dict[str, Any]], None, None]],
-    request_ids: Optional[List[str]] = None,
-    console: Optional[Console] = None,
-    layout_type: LAYOUT_TYPES = "vertical"
-) -> List[Tuple[str, Optional[Dict[str, Any]]]]:
+    titles: List[str],
+    layout: str = "horizontal",
+    console: Optional[Console] = None
+) -> List[str]:
     """
-    多流并行输出处理器
+    Render multiple streams with Rich
     
     Args:
-        stream_generators: 流处理器列表
-        request_ids: 对应请求ID列表
-        console: Rich Console对象
-        layout_type: 布局类型 vertical/horizontal
+        stream_generators: List of stream generators
+        titles: List of titles for each stream
+        layout: "horizontal" or "vertical"
+        console: Optional Rich console instance
         
     Returns:
-        List[Tuple[str, Dict]]: 各流的处理结果
+        List of final content from each stream
     """
-    # 确保使用统一的console实例
-    if console is None:
-        console = Console(force_terminal=True, color_system="auto", height=24)
-    
-    # 初始化控制器
-    controller = StreamController(layout_type, console=console)
-    stream_count = len(stream_generators)
-    controller.prepare_layout(stream_count)
-    
-    # 启动工作线程
-    results = [None] * stream_count
-    threads = []
-    
-    # 创建工作线程
-    def worker_target(idx: int, gen: Generator[Tuple[str, Dict[str, Any]], None, None]):
-        req_id = request_ids[idx] if request_ids and idx < len(request_ids) else None
-        results[idx] = stream_worker(idx, gen, controller, req_id)
-    
-    # 启动所有工作线程
-    for idx, gen in enumerate(stream_generators):
-        t = Thread(target=worker_target, args=(idx, gen))
-        t.start()
-        threads.append(t)
-    
-    # 主渲染线程
-    try:
-        with Live(
-            controller.layout, 
-            console=console or controller.console,
-            refresh_per_second=10,
-            screen=True
-        ) as live:
-            while controller.running:
-                updated = False
-                try:
-                    while True:  # 处理队列中的所有更新
-                        idx, content, final = controller.queue.get_nowait()
-                        controller.update_panel(idx, content, final)
-                        updated = True
-                except Empty:
-                    pass
-                
-                if updated:
-                    live.refresh()
-                
-                # 检查线程是否全部完成
-                if all(not t.is_alive() for t in threads):
-                    break
-                
-                time.sleep(0.1)
-                
-    finally:
-        controller.running = False
-        for t in threads:
-            t.join()
+    renderer = MultiStreamRenderer(titles, layout, console)
+    return renderer.render_streams(stream_generators)
 
-    # 确保最后一次刷新
-    (console or controller.console).print(controller.layout)        
-    return results
 
 def stream_out(
     stream_generator: Generator[Tuple[str, Dict[str, Any]], None, None],
@@ -358,6 +234,8 @@ def stream_out(
             title="Error",
             border_style="red"
         ))
+        # import traceback
+        # traceback.print_exc()
         
         if request_id and request_queue:
             request_queue.add_request(

autocoder/utils/rest.py

@@ -1,16 +1,22 @@
 import requests
 from bs4 import BeautifulSoup
-from typing import List,Dict,Type,Optional
+from typing import List,Dict,Union,Optional
 from autocoder.common import SourceCode
 import byzerllm
-from bs4 import BeautifulSoup
 from loguru import logger
 import os
 from pathlib import Path
 from autocoder.common import files as FileUtils
+import traceback
+from autocoder.rag.loaders import (
+    extract_text_from_pdf,
+    extract_text_from_docx,
+    extract_text_from_ppt,
+    extract_text_from_excel
+)
 
 class HttpDoc:
-    def __init__(self, args, llm: byzerllm.ByzerLLM,urls:Optional[List[str]]=None):
+    def __init__(self, args, llm: Union[byzerllm.ByzerLLM, byzerllm.SimpleByzerLLM],urls:Optional[List[str]]=None):
         self.args = args
         urls_from_args = self.args.urls
         if urls_from_args:
@@ -41,104 +47,50 @@ class HttpDoc:
         {{ html }}
                 
         输出的内容请以 "<MARKER></MARKER> 标签对包裹。
-        """
-
-    def is_binary_file(self,filepath):            
-        try:
-            with open(filepath, 'rb') as file:
-                chunk = file.read(1024*8)  # Read first 1024 bytes
-                if b'\x00' in chunk:  # Binary files often contain null bytes
-                    return True
-                # Attempt to decode as UTF-8 (or any encoding you expect your text files to be in)
-                chunk.decode('utf-8')
-                return False
-        except UnicodeDecodeError:
-            return True   
+        """    
 
-    def get_file_extractor(self):        
+    def _process_local_file(self, file_path: str) -> List[SourceCode]:
+        """统一处理本地文件，返回标准化的 SourceCode 列表"""
+        results = []
         try:
-            from llama_index.core.readers.base import BaseReader
-            from fsspec import AbstractFileSystem
-            from llama_index.core.schema import Document
-            from llama_index.core.readers.file.base import get_default_fs
-            from llama_index.readers.file import (
-                DocxReader,
-                EpubReader,
-                HWPReader,
-                ImageReader,
-                IPYNBReader,
-                MarkdownReader,
-                MboxReader,
-                PandasCSVReader,
-                PDFReader,
-                PptxReader,
-                VideoAudioReader,
-            )  # pants: no-infer-dep                                
-        except ImportError as e:
-            raise ImportError(f"`llama-index-readers-file` package not found. {e}")
+            ext = os.path.splitext(file_path)[1].lower()
+                        
+            # 分发到不同 loader
+            if ext == '.pdf':
+                content = extract_text_from_pdf(file_path)
+                results.append(SourceCode(module_name=file_path, source_code=content))
+            elif ext == '.docx':
+                content = extract_text_from_docx(file_path)
+                results.append(SourceCode(module_name=file_path, source_code=content))
+            elif ext in ('.pptx', '.ppt'):
+                for slide_id, slide_content in extract_text_from_ppt(file_path):
+                    results.append(SourceCode(module_name=f"{file_path}#{slide_id}", source_code=slide_content))
+            elif ext in ('.xlsx', '.xls'):
+                for sheet_name, sheet_content in extract_text_from_excel(file_path):
+                    results.append(SourceCode(module_name=f"{file_path}#{sheet_name}", source_code=sheet_content))
+            else:
+                content = FileUtils.read_file(file_path)
+                results.append(SourceCode(module_name=file_path, source_code=content))
 
-        default_file_reader_cls: Dict[str, BaseReader] = {
-            ".hwp": HWPReader(),
-            ".pdf": PDFReader(return_full_document=True),
-            ".docx": DocxReader(),
-            # ".pptx": PptxReader(),
-            # ".ppt": PptxReader(),
-            # ".pptm": PptxReader(),
-            # ".jpg": ImageReader(),
-            # ".png": ImageReader(),
-            # ".jpeg": ImageReader(),
-            # ".mp3": VideoAudioReader(),
-            # ".mp4": VideoAudioReader(),
-            # ".csv": PandasCSVReader(),
-            ".epub": EpubReader(),            
-            ".mbox": MboxReader(),
-            ".ipynb": IPYNBReader(),            
-        }
-        return default_file_reader_cls
+        except Exception as e:
+            logger.error(f"Failed to process {file_path}: {str(e)}")
+            traceback.print_exc()
+        
+        return results
 
     def crawl_urls(self) -> List[SourceCode]:
-        source_codes = []        
+        source_codes = []
         for url in self.urls:
-            if not url.startswith("http://") and not url.startswith("https://"):
+            if not url.startswith(("http://", "https://")):
                 try:
-                 from llama_index.core import SimpleDirectoryReader
-                 exts = self.get_file_extractor()
-                 documents = []   
-
-                 def process_single_file(file_path: str,skip_binary_file_test:bool=False): 
-                    temp_documents = []
-                    ext = os.path.splitext(file_path)[1].lower()                    
-                    if  not skip_binary_file_test and self.is_binary_file(file_path):
-                        logger.warning(f"Skipping binary file: {file_path}")
-                        return temp_documents
-                    
-                    if ext not in exts.keys():                                                
-                        main_content = FileUtils.read_file(file_path)
-                        source_code = SourceCode(module_name=file_path, source_code=main_content)
-                        source_codes.append(source_code)                                   
+                    if os.path.isdir(url):
+                        for root, _, files in os.walk(url, followlinks=True):
+                            for file in files:
+                                source_codes.extend(self._process_local_file(os.path.join(root, file)))
                     else:
-                        temp_documents = SimpleDirectoryReader(input_files=[url],file_extractor=exts).load_data()  
-                    return temp_documents                                    
-
-                 if os.path.isdir(url):
-                    for root, dirs, files in os.walk(url,followlinks=True):
-                        dirs[:] = [d for d in dirs if d not in ['.git',"node_modules"]]  # Exclude .git directory
-                        for file in files:
-                            file_path = os.path.join(root, file)                            
-                            documents.extend(process_single_file(file_path))
-                    
-                 else:                    
-                    documents.extend(process_single_file(url,skip_binary_file_test=True))
-                    
-                 for document in documents:
-                    source_code = SourceCode(module_name=document.metadata["file_path"], source_code=document.get_content())
-                    source_codes.append(source_code)
-
-                except ImportError as e:
-                    logger.warning(f"Failed to import llama_index. Please install it using 'pip install llama_index' {e}")
-                    main_content = FileUtils.read_file(url)
-                    source_code = SourceCode(module_name=url, source_code=main_content)
-                    source_codes.append(source_code)                                                
+                        source_codes.extend(self._process_local_file(url))
+                except Exception as e:
+                    logger.error(f"Error accessing path {url}: {str(e)}")
             else:
                 if self.args.urls_use_model:
                     from autocoder.common.screenshots import gen_screenshots

autocoder/version.py

@@ -1 +1 @@
-__version__ = "0.1.243"
+__version__ = "0.1.245"