auto-coder 0.1.309__tar.gz → 0.1.312__tar.gz

{auto_coder-0.1.309 → auto_coder-0.1.312}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.309
+Version: 0.1.312
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -28,7 +28,7 @@ Requires-Dist: prompt-toolkit
 Requires-Dist: tokenizers
 Requires-Dist: aiofiles
 Requires-Dist: readerwriterlock
-Requires-Dist: byzerllm[saas]>=0.1.177
+Requires-Dist: byzerllm[saas]>=0.1.178
 Requires-Dist: patch
 Requires-Dist: diff_match_patch
 Requires-Dist: GitPython

{auto_coder-0.1.309 → auto_coder-0.1.312}/src/auto_coder.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.309
+Version: 0.1.312
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -28,7 +28,7 @@ Requires-Dist: prompt-toolkit
 Requires-Dist: tokenizers
 Requires-Dist: aiofiles
 Requires-Dist: readerwriterlock
-Requires-Dist: byzerllm[saas]>=0.1.177
+Requires-Dist: byzerllm[saas]>=0.1.178
 Requires-Dist: patch
 Requires-Dist: diff_match_patch
 Requires-Dist: GitPython

{auto_coder-0.1.309 → auto_coder-0.1.312}/src/auto_coder.egg-info/SOURCES.txt

@@ -80,6 +80,7 @@ src/autocoder/common/mcp_server.py
 src/autocoder/common/mcp_tools.py
 src/autocoder/common/memory_manager.py
 src/autocoder/common/model_speed_test.py
+src/autocoder/common/openai_content.py
 src/autocoder/common/printer.py
 src/autocoder/common/recall_validation.py
 src/autocoder/common/result_manager.py
@@ -91,6 +92,7 @@ src/autocoder/common/stats_panel.py
 src/autocoder/common/stream_out_type.py
 src/autocoder/common/sys_prompt.py
 src/autocoder/common/text.py
+src/autocoder/common/token_cost_caculate.py
 src/autocoder/common/types.py
 src/autocoder/common/utils_code_auto_generate.py
 src/autocoder/common/mcp_servers/__init__.py

{auto_coder-0.1.309 → auto_coder-0.1.312}/src/auto_coder.egg-info/requires.txt

@@ -18,7 +18,7 @@ prompt-toolkit
 tokenizers
 aiofiles
 readerwriterlock
-byzerllm[saas]>=0.1.177
+byzerllm[saas]>=0.1.178
 patch
 diff_match_patch
 GitPython

{auto_coder-0.1.309 → auto_coder-0.1.312}/src/autocoder/auto_coder_runner.py

@@ -1854,12 +1854,14 @@ def active_context(query: str):
             return
         
         # 创建表格
-        table = Table(title="活动上下文任务列表")
-        table.add_column("任务ID", style="cyan")
-        table.add_column("状态", style="green")
-        table.add_column("开始时间", style="yellow")
-        table.add_column("完成时间", style="yellow")
-        table.add_column("文件", style="blue")
+        table = Table(title="活动上下文任务列表", show_lines=True, expand=True)
+        table.add_column("任务ID", style="cyan", no_wrap=False)
+        table.add_column("状态", style="green", no_wrap=False)
+        table.add_column("开始时间", style="yellow", no_wrap=False)
+        table.add_column("完成时间", style="yellow", no_wrap=False)
+        table.add_column("文件", style="blue", no_wrap=False)
+        table.add_column("Token统计", style="magenta", no_wrap=False)
+        table.add_column("费用", style="red", no_wrap=False)
         
         # 添加任务数据
         for task in all_tasks:
@@ -1887,17 +1889,29 @@ def active_context(query: str):
             # 获取文件名
             file_name = task.get("file_name", "未知")
             
+            # 获取token信息
+            total_tokens = task.get("total_tokens", 0)
+            input_tokens = task.get("input_tokens", 0)
+            output_tokens = task.get("output_tokens", 0)
+            token_info = f"总计: {total_tokens:,}\n输入: {input_tokens:,}\n输出: {output_tokens:,}"
+            
+            # 获取费用信息
+            cost = task.get("cost", 0.0)
+            cost_info = f"${cost:.6f}"
+            
             # 添加到表格
             table.add_row(
                 task.get("task_id", "未知"),
                 status_display,
                 start_time_str,
                 completion_time_str,
-                file_name
+                file_name,
+                token_info,
+                cost_info
             )
         
         # 显示表格
-        console = Console()
+        console = Console(width=120)  # 设置更宽的显示宽度
         console.print(table)
 
 

auto_coder-0.1.312/src/autocoder/common/openai_content.py

@@ -0,0 +1,256 @@
+from typing import Any, Dict, List, Optional, Union
+import base64
+import os
+from enum import Enum
+from pydantic import BaseModel, Field, validator
+
+
+class ContentType(str, Enum):
+    """Type of content in the OpenAI chat message."""
+    TEXT = "text"
+    IMAGE_URL = "image_url"
+
+
+class ImageUrl(BaseModel):
+    """Image URL structure in OpenAI chat messages."""
+    url: str = Field(..., description="URL of the image, can be http(s) or data URI")
+
+    @validator('url')
+    def validate_url(cls, v):
+        """Validate that URL is either an http(s) URL or a valid data URI."""
+        if v.startswith(('http://', 'https://')):
+            return v
+        elif v.startswith('data:image/'):
+            return v
+        else:
+            raise ValueError("Image URL must be http(s) or data URI format")
+
+
+class TextContent(BaseModel):
+    """Text content in OpenAI chat messages."""
+    type: str = ContentType.TEXT
+    text: str
+
+
+class ImageUrlContent(BaseModel):
+    """Image URL content in OpenAI chat messages."""
+    type: str = ContentType.IMAGE_URL
+    image_url: Union[str, ImageUrl]
+
+    @validator('image_url')
+    def validate_image_url(cls, v):
+        """Convert string to ImageUrl if necessary."""
+        if isinstance(v, str):
+            return ImageUrl(url=v)
+        return v
+
+
+ContentItem = Union[TextContent, ImageUrlContent]
+
+
+class OpenAIMessage(BaseModel):
+    """Model for an OpenAI chat message."""
+    role: str
+    content: Union[str, List[ContentItem]]
+    name: Optional[str] = None
+
+
+class OpenAIConversation(BaseModel):
+    """Model for a conversation with OpenAI."""
+    messages: List[OpenAIMessage]
+
+
+def is_structured_content(content: Any) -> bool:
+    """
+    Check if the content is structured (list of items with type field).
+    
+    Args:
+        content: The content to check
+        
+    Returns:
+        bool: True if the content is structured, False otherwise
+    """
+    if not isinstance(content, list):
+        return False
+    
+    if not content:
+        return False
+    
+    # Check if all items have a 'type' field
+    return all(isinstance(item, dict) and 'type' in item for item in content)
+
+
+def encode_image_to_base64(image_path: str) -> str:
+    """
+    Encode an image file to base64.
+    
+    Args:
+        image_path: Path to the image file
+        
+    Returns:
+        str: Base64-encoded image data
+    """
+    if not os.path.exists(image_path):
+        raise FileNotFoundError(f"Image file not found: {image_path}")
+    
+    with open(image_path, "rb") as image_file:
+        encoded_string = base64.b64encode(image_file.read()).decode('utf-8')
+    
+    # Determine content type based on file extension
+    file_ext = os.path.splitext(image_path)[1].lower()
+    content_type = {
+        '.jpg': 'image/jpeg',
+        '.jpeg': 'image/jpeg',
+        '.png': 'image/png',
+        '.gif': 'image/gif',
+        '.webp': 'image/webp',
+    }.get(file_ext, 'image/jpeg')
+    
+    return f"data:{content_type};base64,{encoded_string}"
+
+
+def create_text_content(text: str) -> TextContent:
+    """
+    Create a text content item.
+    
+    Args:
+        text: The text content
+        
+    Returns:
+        TextContent: A text content item
+    """
+    return TextContent(text=text)
+
+
+def create_image_content(image_path_or_url: str) -> ImageUrlContent:
+    """
+    Create an image content item from a file path or URL.
+    
+    Args:
+        image_path_or_url: Path to the image file or an image URL
+        
+    Returns:
+        ImageUrlContent: An image content item
+    """
+    # If it's a URL already, use it directly
+    if image_path_or_url.startswith(('http://', 'https://', 'data:')):
+        return ImageUrlContent(image_url=image_path_or_url)
+    
+    # Otherwise, treat it as a file path and encode it
+    return ImageUrlContent(image_url=encode_image_to_base64(image_path_or_url))
+
+
+def normalize_content(content: Any) -> Union[str, List[ContentItem]]:
+    """
+    Normalize content to either a string or a list of structured content items.
+    
+    Args:
+        content: The content to normalize
+        
+    Returns:
+        Union[str, List[ContentItem]]: Normalized content
+    """
+    if isinstance(content, str):
+        return content
+    
+    if is_structured_content(content):
+        normalized_items = []
+        for item in content:
+            if item['type'] == ContentType.TEXT:
+                normalized_items.append(create_text_content(item['text']))
+            elif item['type'] == ContentType.IMAGE_URL:
+                normalized_items.append(ImageUrlContent(image_url=item['image_url']))
+        return normalized_items
+    
+    # If it's neither a string nor structured content, convert to string
+    return str(content)
+
+
+def create_message(role: str, content: Union[str, List[ContentItem]], name: Optional[str] = None) -> OpenAIMessage:
+    """
+    Create an OpenAI chat message.
+    
+    Args:
+        role: Role of the message sender (system, user, assistant)
+        content: Content of the message (string or structured content)
+        name: Optional name of the sender
+        
+    Returns:
+        OpenAIMessage: An OpenAI chat message
+    """
+    return OpenAIMessage(
+        role=role,
+        content=normalize_content(content),
+        name=name
+    )
+
+
+def process_conversations(conversations: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """
+    处理会话列表，确保每个消息都符合标准格式，并只保留文本内容。
+    
+    Args:
+        conversations: 会话列表，可能包含各种格式的消息
+        
+    Returns:
+        List[Dict[str, Any]]: 标准化后的会话列表，每个消息都有 role 和 content 字段
+    
+    例子:
+        >>> conversations = [
+        ...     {"role": "user", "content": "Hello"},
+        ...     {"role": "assistant", "content": "Hi, how can I help?"},
+        ...     {"role": "user", "content": [
+        ...         {"type": "text", "text": "What's in this image?"},
+        ...         {"type": "image_url", "image_url": "data:image/jpeg;base64,/9j/4AAQ..."}
+        ...     ]}
+        ... ]
+        >>> processed = process_conversations(conversations)
+        >>> # 结果保持相同的结构，但确保格式一致性
+
+        输出格式要是这样的：
+        [
+            {"role": "user", "content": "Hello"},
+            {"role": "assistant", "content": "Hi, how can I help?"},
+            {"role": "user", "content": "What's in this image?"}         
+        ]
+
+        只保留 text 内容。如果有多个 text 内容，用换行符连接弄成一个。
+        
+    """
+    processed_conversations = []
+    
+    for message in conversations:
+        # 确保消息有 role 字段
+        if "role" not in message:
+            raise ValueError(f"Message missing 'role' field: {message}")
+        
+        role = message["role"]
+        
+        # 处理 content 字段，确保存在
+        if "content" not in message:
+            processed_content = ""  # 如果不存在，设置为空字符串
+        else:
+            content = message["content"]
+            
+            # 处理结构化内容
+            if isinstance(content, list) and is_structured_content(content):
+                # 提取所有文本内容并用换行符连接
+                text_contents = []
+                for item in content:
+                    if item.get('type') == ContentType.TEXT and 'text' in item:
+                        text_contents.append(item['text'])
+                processed_content = '\n'.join(text_contents)
+            else:
+                # 如果是字符串或其他类型，确保转换为字符串
+                processed_content = str(content) if content is not None else ""
+        
+        # 构建标准化的消息
+        processed_message = {"role": role, "content": processed_content}
+        
+        # 如果原消息有 name 字段，也加入
+        if "name" in message and message["name"]:
+            processed_message["name"] = message["name"]
+        
+        processed_conversations.append(processed_message)
+    
+    return processed_conversations 

auto_coder-0.1.312/src/autocoder/common/token_cost_caculate.py

@@ -0,0 +1,200 @@
+from typing import Dict, Any, Optional, Tuple, List
+import time
+from loguru import logger
+import byzerllm
+from byzerllm import MetaHolder
+from autocoder.utils import llms
+from autocoder.events.event_manager_singleton import get_event_manager
+from autocoder.events import event_content as EventContentCreator
+from pydantic import BaseModel, Field
+
+
+class TokenUsageStats(BaseModel):
+    """Token使用统计数据模型"""
+    input_tokens: int = Field(default=0, description="输入的token数量")
+    output_tokens: int = Field(default=0, description="输出的token数量")
+    input_cost: float = Field(default=0.0, description="输入token的成本")
+    output_cost: float = Field(default=0.0, description="输出token的成本")
+    
+    @property
+    def total_tokens(self) -> int:
+        """总token数"""
+        return self.input_tokens + self.output_tokens
+    
+    @property
+    def total_cost(self) -> float:
+        """总成本"""
+        return self.input_cost + self.output_cost
+
+
+class TokenCostCalculator:
+    """
+    Token计费计算器 - 提供获取模型信息、计算token成本和记录token统计信息的公共功能
+    """
+
+    def __init__(self, logger_name: str = "TokenCostCalculator"):
+        """
+        初始化Token计费计算器
+
+        Args:
+            logger_name: logger名称，默认为TokenCostCalculator            
+        """
+        self.logger = logger.bind(name=logger_name)
+
+    def get_model_info(self, llm: byzerllm.ByzerLLM,product_mode:str="lite") -> Tuple[str, Dict[str, Dict[str, float]]]:
+        """
+        获取模型名称和价格信息
+
+        Args:
+            llm: ByzerLLM实例
+
+        Returns:
+            Tuple[str, Dict]: 模型名称字符串和价格信息字典
+        """
+        # 获取模型名称列表
+        model_names = llms.get_llm_names(llm)
+        model_name = ",".join(model_names)
+
+        # 获取模型价格信息
+        model_info_map = {}
+        for name in model_names:
+            info = llms.get_model_info(name, product_mode)
+            if info:
+                model_info_map[name] = {
+                    "input_price": info.get("input_price", 0.0),
+                    "output_price": info.get("output_price", 0.0)
+                }
+
+        return model_name, model_info_map
+
+    def calculate_token_costs(self, meta_holder: MetaHolder,
+                              model_info_map: Dict[str, Dict[str, float]]) -> TokenUsageStats:
+        """
+        计算token统计和成本
+
+        Args:
+            meta_holder: MetaHolder实例
+            model_info_map: 模型价格信息字典
+
+        Returns:
+            TokenUsageStats: Token使用统计数据对象
+        """
+        stats = TokenUsageStats()
+
+        if meta_holder.get_meta():
+            meta_dict = meta_holder.get_meta()
+            stats.input_tokens = meta_dict.get("input_tokens_count", 0)
+            stats.output_tokens = meta_dict.get("generated_tokens_count", 0)
+
+            for name, info in model_info_map.items():
+                stats.input_cost += (stats.input_tokens *
+                                     info.get("input_price", 0.0)) / 1000000
+                stats.output_cost += (stats.output_tokens *
+                                      info.get("output_price", 0.0)) / 1000000
+
+        return stats
+
+    def log_event(self,
+                  model_name: str,
+                  operation_name: str,
+                  start_time: float,
+                  end_time: float,
+                  stats: TokenUsageStats,
+                  event_file: Optional[str] = None):
+        """
+        记录token统计信息到日志和事件系统
+
+        Args:
+            model_name: 模型名称
+            operation_name: 操作名称，用于日志记录
+            start_time: 开始时间
+            end_time: 结束时间
+            stats: Token使用统计数据
+            event_file: 事件文件路径，如果提供则写入事件
+        """
+        elapsed_time = end_time - start_time
+
+        # 计算token生成速度（每秒生成的token数）
+        speed = stats.output_tokens / elapsed_time if elapsed_time > 0 else 0
+
+        # 记录日志
+        self.logger.info(f"{operation_name} Stats - Model: {model_name}, "
+                    f"Input Tokens: {stats.input_tokens}, "
+                    f"Output Tokens: {stats.output_tokens}, "
+                    f"Total Tokens: {stats.total_tokens}, "
+                    f"Time: {elapsed_time:.2f}s, "
+                    f"Speed: {speed:.2f}tokens/s, "
+                    f"Total Cost: ${stats.total_cost:.6f}")
+
+        # 记录事件（如果提供了event_file）
+        try:
+            if event_file:
+                get_event_manager(event_file).write_result(
+                    EventContentCreator.create_result(content=EventContentCreator.ResultTokenStatContent(
+                        model_name=model_name,
+                        elapsed_time=elapsed_time,
+                        first_token_time=0,  # 不跟踪首个token时间
+                        input_tokens=stats.input_tokens,
+                        output_tokens=stats.output_tokens,
+                        input_cost=stats.input_cost,
+                        output_cost=stats.output_cost,
+                        speed=speed
+                    ).to_dict()))
+            else:
+                # 尝试使用默认事件管理器
+                get_event_manager().write_result(
+                    EventContentCreator.create_result(content=EventContentCreator.ResultTokenStatContent(
+                        model_name=model_name,
+                        elapsed_time=elapsed_time,
+                        input_tokens=stats.input_tokens,
+                        output_tokens=stats.output_tokens,
+                        input_cost=stats.input_cost,
+                        output_cost=stats.output_cost,
+                        speed=speed
+                    ).to_dict()))
+        except Exception as e:
+            self.logger.error(f"Error writing event for {operation_name}: {e}")
+
+    def track_token_usage(self,
+                          llm: byzerllm.ByzerLLM,
+                          meta_holder: MetaHolder,
+                          operation_name: str,
+                          start_time: float,
+                          end_time: float,
+                          product_mode: Optional[str] = None,
+                          event_file: Optional[str] = None) -> TokenUsageStats:
+        """
+        跟踪token使用情况的便捷方法，整合了获取模型信息、计算成本和记录统计的功能
+
+        Args:
+            llm: ByzerLLM实例
+            meta_holder: MetaHolder实例
+            operation_name: 操作名称
+            start_time: 开始时间
+            end_time: 结束时间
+            product_mode: 产品模式，如果不提供则使用实例的product_mode
+            event_file: 事件文件路径
+
+        Returns:
+            TokenUsageStats: Token使用统计数据对象
+        """
+        # 使用传入的product_mode或实例的product_mode
+        actual_product_mode = product_mode
+        
+        # 获取模型信息
+        model_name, model_info_map = self.get_model_info(llm,actual_product_mode)
+
+        # 计算token统计和成本
+        stats = self.calculate_token_costs(meta_holder, model_info_map)
+
+        # 记录事件
+        self.log_event(
+            model_name=model_name,
+            operation_name=operation_name,
+            start_time=start_time,
+            end_time=end_time,
+            stats=stats,
+            event_file=event_file
+        )
+
+        return stats