auto-coder 1.0.0__py3-none-any.whl → 2.0.1__py3-none-any.whl

autocoder/rag/terminal/formatters/base.py

@@ -0,0 +1,70 @@
+"""
+格式化器基类和工具函数
+"""
+
+
+def format_rag_output(result_generator, contexts, output_format: str):
+    """
+    统一的RAG输出格式化接口
+    
+    Args:
+        result_generator: RAG返回的生成器 (产生 (chunk, meta) 元组)
+        contexts: 使用的上下文文档列表
+        output_format: 输出格式 (text/json/stream-json)
+    """
+    # 延迟导入避免循环依赖
+    if output_format == "text":
+        from .text import TextFormatter
+        formatter = TextFormatter()
+    elif output_format == "json":
+        from .json_format import JSONFormatter
+        formatter = JSONFormatter()
+    elif output_format == "stream-json":
+        from .stream_json import StreamJSONFormatter
+        formatter = StreamJSONFormatter()
+    else:
+        raise ValueError(f"不支持的输出格式: {output_format}")
+    
+    formatter.format(result_generator, contexts)
+
+
+def detect_stage_from_reasoning(reasoning_content: str) -> str:
+    """
+    根据 reasoning_content 检测当前处理阶段
+
+    Args:
+        reasoning_content: 推理内容（通常包含阶段提示信息）
+
+    Returns:
+        阶段类型: retrieval/filtering/chunking/generation/processing
+    """
+    if not reasoning_content:
+        return "processing"
+
+    content_lower = reasoning_content.lower()
+
+    # 检测文档检索阶段
+    if (
+        "rag_processing" in content_lower
+        or "searching" in content_lower
+        or "搜索" in reasoning_content
+    ):
+        return "retrieval"
+
+    # 检测文档过滤阶段
+    if "filter" in content_lower or "过滤" in reasoning_content:
+        return "filtering"
+
+    # 检测文档分块阶段
+    if "chunk" in content_lower or "分块" in reasoning_content:
+        return "chunking"
+
+    # 检测答案生成阶段
+    if (
+        "thinking" in content_lower
+        or "send_to_model" in reasoning_content
+        or "model" in reasoning_content
+    ):
+        return "generation"
+
+    return "processing"

autocoder/rag/terminal/formatters/json_format.py

@@ -0,0 +1,66 @@
+"""
+JSON 格式化器
+
+一次性输出完整的 JSON 结果，包含答案、上下文、处理阶段和元数据。
+"""
+
+import json
+from .base import detect_stage_from_reasoning
+
+
+class JSONFormatter:
+    """JSON格式化器：收集所有内容后一次性输出完整的JSON"""
+
+    def format(self, result_generator, contexts):
+        """
+        格式化为JSON输出
+
+        Args:
+            result_generator: 生成器，产生 (chunk, meta) 元组
+            contexts: 上下文文档列表
+        """
+        answer = ""
+        stages = []
+        final_metadata = {}
+        current_stage_type = None
+
+        for chunk, meta in result_generator:
+            # 收集答案内容
+            answer += chunk
+
+            # 收集元数据
+            if meta:
+                final_metadata = {
+                    "input_tokens": meta.input_tokens_count,
+                    "generated_tokens": meta.generated_tokens_count,
+                    "finish_reason": meta.finish_reason,
+                }
+
+                # 收集中间处理步骤（从 reasoning_content）
+                if meta.reasoning_content:
+                    stage_type = detect_stage_from_reasoning(meta.reasoning_content)
+
+                    # 避免重复记录相同阶段
+                    if stage_type != current_stage_type:
+                        stages.append(
+                            {
+                                "type": stage_type,
+                                "message": meta.reasoning_content,
+                                "tokens": {
+                                    "input": meta.input_tokens_count,
+                                    "generated": meta.generated_tokens_count,
+                                },
+                            }
+                        )
+                        current_stage_type = stage_type
+
+        # 构建完整的结果
+        result = {
+            "answer": answer,
+            "contexts": contexts,
+            "stages": stages,
+            "metadata": final_metadata,
+        }
+
+        # 一次性输出JSON
+        print(json.dumps(result, ensure_ascii=False, indent=2))

autocoder/rag/terminal/formatters/stream_json.py

@@ -0,0 +1,95 @@
+"""
+流式 JSON 格式化器
+
+实时输出每个处理步骤的 JSON 事件，适合前端集成和实时进度显示。
+"""
+
+import json
+from datetime import datetime
+from .base import detect_stage_from_reasoning
+
+
+class StreamJSONFormatter:
+    """流式JSON格式化器：实时输出每个事件为一行JSON"""
+
+    def format(self, result_generator, contexts):
+        """
+        格式化为流式JSON输出
+
+        每个事件输出一行JSON，格式：
+        {"event_type": "...", "data": {...}, "timestamp": "..."}
+
+        Args:
+            result_generator: 生成器，产生 (chunk, meta) 元组
+            contexts: 上下文文档列表
+        """
+        current_stage_type = None
+        final_metadata = {}
+
+        # 1. 发送开始事件
+        self._emit_event("start", {"status": "started"})
+
+        try:
+            # 2. 处理每个chunk和meta
+            for chunk, meta in result_generator:
+                # 处理元数据中的中间步骤信息
+                if meta:
+                    # 更新最终元数据
+                    final_metadata = {
+                        "input_tokens": meta.input_tokens_count,
+                        "generated_tokens": meta.generated_tokens_count,
+                        "finish_reason": meta.finish_reason,
+                    }
+
+                    # 处理 reasoning_content（各个处理阶段）
+                    if meta.reasoning_content:
+                        stage_type = detect_stage_from_reasoning(meta.reasoning_content)
+
+                        # 避免重复发送相同阶段（但message不同时也要发送）
+                        if (
+                            stage_type != current_stage_type
+                            or meta.reasoning_content.strip()
+                        ):
+                            self._emit_event(
+                                "stage",
+                                {
+                                    "type": stage_type,
+                                    "message": meta.reasoning_content,
+                                    "tokens": {
+                                        "input": meta.input_tokens_count,
+                                        "generated": meta.generated_tokens_count,
+                                    },
+                                },
+                            )
+                            current_stage_type = stage_type
+
+                # 处理实际的答案内容
+                if chunk:
+                    self._emit_event("content", {"content": chunk})
+
+            # 3. 发送上下文信息
+            if contexts:
+                self._emit_event("contexts", {"contexts": contexts})
+
+            # 4. 发送结束事件（包含最终元数据）
+            self._emit_event("end", {"status": "completed", "metadata": final_metadata})
+
+        except Exception as e:
+            # 发送错误事件
+            self._emit_event("error", {"error": str(e)})
+            raise
+
+    def _emit_event(self, event_type: str, data: dict):
+        """
+        输出单个事件（一行JSON）
+
+        Args:
+            event_type: 事件类型
+            data: 事件数据
+        """
+        event = {
+            "event_type": event_type,
+            "data": data,
+            "timestamp": datetime.now().isoformat(),
+        }
+        print(json.dumps(event, ensure_ascii=False), flush=True)

autocoder/rag/terminal/formatters/text.py

@@ -0,0 +1,28 @@
+"""
+文本格式化器
+
+纯文本输出，只输出最终答案，适合人类阅读和管道传递。
+"""
+
+
+class TextFormatter:
+    """文本格式化器：只输出答案内容，不包含中间步骤"""
+
+    def format(self, result_generator, contexts):
+        """
+        格式化为纯文本输出
+
+        Args:
+            result_generator: 生成器，产生 (chunk, meta) 元组
+            contexts: 上下文文档列表（在text格式中不输出）
+        """
+        for chunk, meta in result_generator:
+            # 只输出实际的答案内容（chunk）
+            # 忽略 meta.reasoning_content（中间处理步骤）
+            if meta.reasoning_content:
+                print(meta.reasoning_content, end="", flush=True)
+            if chunk:
+                print(chunk, end="", flush=True)
+
+        # 输出完成后换行
+        print()

autocoder/rag/terminal/init.py

@@ -0,0 +1,120 @@
+"""系统初始化模块"""
+
+import subprocess
+from prompt_toolkit import prompt
+from prompt_toolkit.shortcuts import radiolist_dialog
+from prompt_toolkit.formatted_text import HTML
+from autocoder.chat_auto_coder_lang import get_message
+
+
+def initialize_system(args):
+    """初始化系统环境（Ray、模型等）"""
+    if args.product_mode == "lite":
+        return
+
+    print(f"\n\033[1;34m{get_message('initializing')}\033[0m")
+
+    def print_status(message, status):
+        if status == "success":
+            print(f"\033[32m✓ {message}\033[0m")
+        elif status == "warning":
+            print(f"\033[33m! {message}\033[0m")
+        elif status == "error":
+            print(f"\033[31m✗ {message}\033[0m")
+        else:
+            print(f"  {message}")
+
+    # Check if Ray is running
+    print_status(get_message("checking_ray"), "")
+    ray_status = subprocess.run(["ray", "status"], capture_output=True, text=True)
+    if ray_status.returncode != 0:
+        print_status(get_message("ray_not_running"), "warning")
+        try:
+            subprocess.run(["ray", "start", "--head"], check=True)
+            print_status(get_message("ray_start_success"), "success")
+        except subprocess.CalledProcessError:
+            print_status(get_message("ray_start_fail"), "error")
+            return
+    else:
+        print_status(get_message("ray_running"), "success")
+
+    # Check if deepseek_chat model is available
+    print_status(get_message("checking_model"), "")
+    try:
+        result = subprocess.run(
+            ["easy-byzerllm", "chat", "v3_chat", "你好"],
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+        if result.returncode == 0:
+            print_status(get_message("model_available"), "success")
+            print_status(get_message("init_complete_final"), "success")
+            return
+    except subprocess.TimeoutExpired:
+        print_status(get_message("model_timeout"), "error")
+    except subprocess.CalledProcessError:
+        print_status(get_message("model_error"), "error")
+
+    # If deepseek_chat is not available, prompt user to choose a provider
+    print_status(get_message("model_not_available"), "warning")
+    choice = radiolist_dialog(
+        title=get_message("provider_selection"),
+        text=get_message("provider_selection"),
+        values=[
+            ("1", "Deepseek官方(https://www.deepseek.com/)"),
+        ],
+    ).run()
+
+    if choice is None:
+        print_status(get_message("no_provider"), "error")
+        return
+
+    api_key = prompt(HTML(f"<b>{get_message('enter_api_key')} </b>"))
+
+    if choice == "1":
+        print_status(get_message("deploying_model").format("Deepseek官方"), "")
+
+        deploy_cmd = [
+            "byzerllm",
+            "deploy",
+            "--pretrained_model_type",
+            "saas/openai",
+            "--cpus_per_worker",
+            "0.001",
+            "--gpus_per_worker",
+            "0",
+            "--worker_concurrency",
+            "1000",
+            "--num_workers",
+            "1",
+            "--infer_params",
+            f"saas.base_url=https://api.deepseek.com/v1 saas.api_key={api_key} saas.model=deepseek-chat",
+            "--model",
+            "v3_chat",
+        ]
+
+    try:
+        subprocess.run(deploy_cmd, check=True)
+        print_status(get_message("deploy_complete"), "success")
+    except subprocess.CalledProcessError:
+        print_status(get_message("deploy_fail"), "error")
+        return
+
+    # Validate the deployment
+    print_status(get_message("validating_deploy"), "")
+    try:
+        validation_result = subprocess.run(
+            ["easy-byzerllm", "chat", "v3_chat", "你好"],
+            capture_output=True,
+            text=True,
+            timeout=30,
+            check=True,
+        )
+        print_status(get_message("validation_success"), "success")
+    except (subprocess.TimeoutExpired, subprocess.CalledProcessError):
+        print_status(get_message("validation_fail"), "error")
+        print_status(get_message("manual_start"), "warning")
+        print_status("easy-byzerllm chat v3_chat 你好", "")
+
+    print_status(get_message("init_complete_final"), "success")

autocoder/rag/terminal/utils.py

@@ -0,0 +1,106 @@
+"""工具函数模块"""
+
+import hashlib
+import os
+from pathlib import Path
+from rich.console import Console
+from rich.table import Table
+from autocoder.rag.utils import process_file_local
+from autocoder.rag.token_counter import TokenCounter
+from autocoder.version import __version__
+
+
+def print_banner():
+    """打印启动横幅"""
+    print(
+        f"""
+    \033[1;32m
+      _     _     __  __       _   _    _  _____ _____     _______   ____      _    ____ 
+     | |   | |   |  \/  |     | \ | |  / \|_   _|_ _\ \   / / ____| |  _ \    / \  / ___|
+     | |   | |   | |\/| |_____|  \| | / _ \ | |  | | \ \ / /|  _|   | |_) |  / _ \| |  _ 
+     | |___| |___| |  | |_____| |\  |/ ___ \| |  | |  \ V / | |___  |  _ <  / ___ \ |_| |
+     |_____|_____|_|  |_|     |_| \_/_/   \_\_| |___|  \_/  |_____| |_| \_\/_/   \_\____|
+                                                                            v{__version__}
+    \033[0m"""
+    )
+
+
+def generate_unique_name_from_path(path: str) -> str:
+    """
+    从路径生成唯一名称（MD5哈希），在规范化后。
+    对于 Linux/Unix 系统，会移除尾部的路径分隔符。
+    """
+    if not path:
+        return ""
+
+    # Normalize the path (resolve absolute path and remove trailing separators)
+    normalized_path = os.path.normpath(os.path.abspath(path))
+
+    # Generate MD5 hash from the normalized path
+    return hashlib.md5(normalized_path.encode("utf-8")).hexdigest()
+
+
+def merge_args_with_config(args, config, arg_class, parser):
+    """
+    合并命令行参数和配置文件参数，优先级如下：
+    1. 命令行参数非默认值，以命令行为准
+    2. 命令行参数为默认值，且配置文件有值，以配置文件为准
+    3. 否则用命令行参数
+    """
+    merged = {}
+    for arg in vars(arg_class()):
+        # 获取默认值
+        try:
+            default = parser.get_default(arg)
+        except Exception:
+            default = None
+
+        if not hasattr(args, arg) and arg not in config:
+            continue
+
+        cli_value = getattr(args, arg, None)
+        config_value = config.get(arg, None)
+
+        # 判断优先级
+        if cli_value != default:
+            merged[arg] = cli_value
+        elif config_value is not None:
+            merged[arg] = config_value
+        else:
+            merged[arg] = cli_value
+
+    return arg_class(**merged)
+
+
+def count_tokens(tokenizer_path: str, file_path: str):
+    """统计文件的 token 数量"""
+    from autocoder.rag.variable_holder import VariableHolder
+    from tokenizers import Tokenizer
+
+    VariableHolder.TOKENIZER_PATH = tokenizer_path
+    VariableHolder.TOKENIZER_MODEL = Tokenizer.from_file(tokenizer_path)
+    token_counter = TokenCounter(tokenizer_path)
+    source_codes = process_file_local(file_path)
+
+    console = Console()
+    table = Table(title="Token Count Results")
+    table.add_column("File", style="cyan")
+    table.add_column("Characters", justify="right", style="magenta")
+    table.add_column("Tokens", justify="right", style="green")
+
+    total_chars = 0
+    total_tokens = 0
+
+    for source_code in source_codes:
+        content = source_code.source_code
+        chars = len(content)
+        tokens = token_counter.count_tokens(content)
+
+        total_chars += chars
+        total_tokens += tokens
+
+        table.add_row(source_code.module_name, str(chars), str(tokens))
+
+    table.add_row("Total", str(total_chars), str(total_tokens), style="bold")
+
+    console.print(table)