auto-coder 0.1.375__tar.gz → 0.1.376__tar.gz

{auto_coder-0.1.375 → auto_coder-0.1.376}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.375
+Version: 0.1.376
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Programming Language :: Python :: 3.10

{auto_coder-0.1.375 → auto_coder-0.1.376}/src/auto_coder.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.375
+Version: 0.1.376
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Programming Language :: Python :: 3.10

{auto_coder-0.1.375 → auto_coder-0.1.376}/src/auto_coder.egg-info/SOURCES.txt

@@ -276,7 +276,6 @@ src/autocoder/rag/long_context_rag.py
 src/autocoder/rag/qa_conversation_strategy.py
 src/autocoder/rag/rag_config.py
 src/autocoder/rag/rag_entry.py
-src/autocoder/rag/raw_rag.py
 src/autocoder/rag/relevant_utils.py
 src/autocoder/rag/searchable.py
 src/autocoder/rag/test_doc_filter.py

{auto_coder-0.1.375 → auto_coder-0.1.376}/src/autocoder/agent/base_agentic/base_agent.py

@@ -511,15 +511,16 @@ class BaseAgent(ABC):
         {%endif%}
 
         # Tool Use Examples
-        {%- set example_count = 0 -%}
-        {%- for tool_tag, example in tool_examples.items() -%}
-        {%- if example -%}
-        {%- set example_count = example_count + 1 -%}
+        {% set example_count = 0 %}
+        {% for tool_tag, example in tool_examples.items() %}
+        {% if example %}
+        {% set example_count = example_count + 1 %}
         ## Example {{ example_count }}: {{ example.title }}
         {{ example.body }}
-        {%- endif -%}
-        {%- endfor -%}                                      
+        {% endif %}
+        {% endfor %}                                      
 
+        
         # Tool Use Guidelines
 
         1. In <thinking> tags, assess what information you already have and what information you need to proceed with the task.
@@ -532,9 +533,9 @@ class BaseAgent(ABC):
         - New terminal output in reaction to the changes, which you may need to consider or act upon.
         - Any other relevant feedback or information related to the tool use.
         6. ALWAYS wait for user confirmation after each tool use before proceeding. Never assume the success of a tool use without explicit confirmation of the result from the user.
-        {%- for tool_name, guideline in tool_guidelines.items() -%}                
+        {% for tool_name, guideline in tool_guidelines.items() %}                
         {{ loop.index + 6 }}. **{{ tool_name }}**: {{ guideline }}
-        {%- endfor -%}
+        {% endfor %}
 
         It is crucial to proceed step-by-step, waiting for the user's message after each tool use before moving forward with the task. This approach allows you to:
         1. Confirm the success of each step before proceeding.

{auto_coder-0.1.375 → auto_coder-0.1.376}/src/autocoder/auto_coder_rag.py

@@ -34,6 +34,7 @@ from autocoder.rag.utils import process_file_local
 import pkg_resources
 from autocoder.rag.token_counter import TokenCounter
 from autocoder.rag.types import RAGServiceInfo
+from autocoder.version import __version__
 
 if platform.system() == "Windows":
     from colorama import init
@@ -169,6 +170,17 @@ def initialize_system(args):
 
 
 def main(input_args: Optional[List[str]] = None):
+    print(
+        f"""
+    \033[1;32m
+      _     _     __  __       _   _    _  _____ _____     _______   ____      _    ____ 
+     | |   | |   |  \/  |     | \ | |  / \|_   _|_ _\ \   / / ____| |  _ \    / \  / ___|
+     | |   | |   | |\/| |_____|  \| | / _ \ | |  | | \ \ / /|  _|   | |_) |  / _ \| |  _ 
+     | |___| |___| |  | |_____| |\  |/ ___ \| |  | |  \ V / | |___  |  _ <  / ___ \ |_| |
+     |_____|_____|_|  |_|     |_| \_/_/   \_\_| |___|  \_/  |_____| |_| \_\/_/   \_\____|
+                                                                            v{__version__}
+    \033[0m"""
+    )
 
     try:
         tokenizer_path = pkg_resources.resource_filename(

{auto_coder-0.1.375 → auto_coder-0.1.376}/src/autocoder/models.py

@@ -165,8 +165,8 @@ def load_models() -> List[Dict]:
         if model.get("api_key_path",""):           
             api_key_file = os.path.join(api_key_dir, model["api_key_path"])
             if os.path.exists(api_key_file):
-                with open(api_key_file, "r",encoding="utf-8") as f:
-                    model["api_key"] = f.read()                   
+                with open(api_key_file, "r", encoding="utf-8") as f:
+                    model["api_key"] = f.read().strip()
     return target_models
 
 def save_models(models: List[Dict]) -> None:

{auto_coder-0.1.375 → auto_coder-0.1.376}/src/autocoder/rag/cache/local_duckdb_storage_cache.py

@@ -10,7 +10,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import List, Dict, Any, Optional, Tuple, Union
 import numpy as np
 from loguru import logger
-from typing import Union
 from byzerllm import SimpleByzerLLM, ByzerLLM
 from autocoder.utils.llms import get_llm_names
 
@@ -31,10 +30,12 @@ from autocoder.rag.cache.base_cache import (
     FileInfo,
     CacheItem,
 )
-from autocoder.rag.utils import process_file_in_multi_process, process_file_local
+from autocoder.rag.utils import (
+    process_file_in_multi_process,
+    process_file_local,
+)
 from autocoder.rag.variable_holder import VariableHolder
-from byzerllm import SimpleByzerLLM, ByzerLLM
-from .failed_files_utils import save_failed_files, load_failed_files
+from .failed_files_utils import save_failed_files
 
 if platform.system() != "Windows":
     import fcntl
@@ -66,7 +67,8 @@ class DuckDBLocalContext:
     def __enter__(self) -> "duckdb.DuckDBPyConnection":
         if not os.path.exists(os.path.dirname(self.database_path)):
             raise ValueError(
-                f"Directory {os.path.dirname(self.database_path)} does not exist."
+                f"Directory {os.path.dirname(self.database_path)} "
+                f"does not exist."
             )
 
         self._conn = duckdb.connect(self.database_path)
@@ -97,10 +99,12 @@ class LocalDuckdbStorage:
         self.persist_dir = persist_dir
         self.cache_dir = os.path.join(self.persist_dir, ".cache")
         self.args = args
-        logger.info(f"正在启动 DuckDBVectorStore.")
+        logger.info("正在启动 DuckDBVectorStore.")
 
         if self.database_name != ":memory:":
-            self.database_path = os.path.join(self.cache_dir, self.database_name)
+            self.database_path = os.path.join(
+                self.cache_dir, self.database_name
+            )
 
         if self.database_name == ":memory:":
             self._conn = duckdb.connect(self.database_name)
@@ -114,7 +118,8 @@ class LocalDuckdbStorage:
             self._conn = None
         logger.info(
             f"DuckDBVectorStore 初始化完成, 存储目录: {self.cache_dir}, "
-            f"数据库名称: {self.database_name}, 数据表名称: {self.table_name}"
+            f"数据库名称: {self.database_name}, "
+            f"数据表名称: {self.table_name}"
         )
 
     @classmethod
@@ -167,15 +172,17 @@ class LocalDuckdbStorage:
                 retry_count += 1
                 if retry_count >= max_retries:
                     logger.error(
-                        f"Failed to get embedding after {max_retries} attempts: {str(e)}"
+                        f"Failed to get embedding after {max_retries} "
+                        f"attempts: {str(e)}"
                     )
                     raise
 
                 # Sleep between 1-5 seconds before retrying
                 sleep_time = 1 + (retry_count * 1.5)
                 logger.warning(
-                    f"Embedding API call failed (attempt {retry_count}/{max_retries}). "
-                    f"Error: {str(e)}. Retrying in {sleep_time:.1f} seconds..."
+                    f"Embedding API call failed (attempt {retry_count}/"
+                    f"{max_retries}). Error: {str(e)}. Retrying in "
+                    f"{sleep_time:.1f} seconds..."
                 )
                 time.sleep(sleep_time)
 
@@ -244,7 +251,9 @@ class LocalDuckdbStorage:
         
         if not context_chunk["raw_content"]:
             context_chunk["raw_content"] = "empty"        
-        context_chunk["raw_content"] = context_chunk["raw_content"][: self.args.rag_emb_text_size]
+        context_chunk["raw_content"] = context_chunk["raw_content"][
+            : self.args.rag_emb_text_size
+        ]
             
         return (
             context_chunk["_id"],
@@ -343,11 +352,14 @@ class LocalDuckDBStorageCache(BaseCacheManager):
         )
         self.queue = []
         self.chunk_size = 1000
-        self.max_output_tokens = extra_params.hybrid_index_max_output_tokens
+        self.max_output_tokens = (
+            extra_params.hybrid_index_max_output_tokens
+        )
 
         # 设置缓存文件路径
         self.cache_dir = os.path.join(self.path, ".cache")
-        self.cache_file = os.path.join(self.cache_dir, "duckdb_storage_speedup.jsonl")
+        self.cache_file = os.path.join(self.cache_dir,
+                                       "duckdb_storage_speedup.jsonl")
         self.cache: Dict[str, CacheItem] = {}
         # 创建缓存目录
         if not os.path.exists(self.cache_dir):
@@ -356,7 +368,9 @@ class LocalDuckDBStorageCache(BaseCacheManager):
         # failed files support
         from .failed_files_utils import load_failed_files
 
-        self.failed_files_path = os.path.join(self.cache_dir, "failed_files.json")
+        self.failed_files_path = os.path.join(
+            self.cache_dir, "failed_files.json"
+        )
         self.failed_files = load_failed_files(self.failed_files_path)
 
         self.lock = threading.Lock()
@@ -406,7 +420,8 @@ class LocalDuckDBStorageCache(BaseCacheManager):
                             continue
                     return cache
             except Exception as e:
-                logger.error(f"Error loading cache file: {str(e)}")
+                logger.warning(f"Error loading cache file: {str(e)}")
+                logger.exception(e)
                 return {}
         return {}
 
@@ -421,7 +436,8 @@ class LocalDuckDBStorageCache(BaseCacheManager):
                         json.dump(cache_item.model_dump(), f, ensure_ascii=False)
                         f.write("\n")
             except IOError as e:
-                logger.error(f"Error writing cache file: {str(e)}")
+                logger.warning(f"Error writing cache file: {str(e)}")
+                logger.exception(e)
         else:
             lock_file = cache_file + ".lock"
             with open(lock_file, "w", encoding="utf-8") as lockf:
@@ -510,11 +526,12 @@ class LocalDuckDBStorageCache(BaseCacheManager):
         self.write_cache()
 
         if items:
-            logger.info("[BUILD CACHE] Clearing existing cache from DuckDB  Storage")
+            logger.info("[BUILD CACHE] Clearing DuckDB Storage cache")
             self.storage.truncate_table()
+            logger.info(f"[BUILD CACHE] Preparing to write to DuckDB Storage.")
             logger.info(
-                f"[BUILD CACHE] Preparing to write to DuckDB  Storage, "
-                f"total chunks: {len(items)}, total files: {len(files_to_process)}"
+                f"[BUILD CACHE] Total chunks: {len(items)}, "
+                f"Total files: {len(files_to_process)}"
             )
 
             # Use a fixed optimal batch size instead of dividing by worker count
@@ -526,9 +543,10 @@ class LocalDuckDBStorageCache(BaseCacheManager):
             total_batches = len(item_batches)
             completed_batches = 0
 
+            logger.info(f"[BUILD CACHE] Writing to DuckDB Storage.")
             logger.info(
-                f"[BUILD CACHE] Starting to write to DuckDB Storage using {batch_size} items per batch, "
-                f"total batches: {total_batches}"
+                f"[BUILD CACHE] Batch size: {batch_size}, "
+                f"Total batches: {total_batches}"
             )
             start_time = time.time()
 
@@ -569,18 +587,26 @@ class LocalDuckDBStorageCache(BaseCacheManager):
                             or (completed_batches == total_batches)
                             or (completed_batches % max(1, total_batches // 10) == 0)
                         ):
+                            progress_percent = (
+                                completed_batches / total_batches * 100
+                                if total_batches > 0
+                                else 0
+                            )
                             logger.info(
-                                f"[BUILD CACHE] Progress: {completed_batches}/{total_batches} batches completed "
-                                f"({(completed_batches / total_batches * 100):.1f}%) "
-                                f"Estimated time remaining: {remaining:.1f}s"
+                                f"[BUILD CACHE] Progress: {completed_batches}/"
+                                f"{total_batches} ({progress_percent:.1f}%). "
+                                f"ETA: {remaining:.1f}s"
                             )
                     except Exception as e:
                         logger.error(f"[BUILD CACHE] Error saving batch: {str(e)}")
                         # Add more detailed error information
+                        batch_len_info = (
+                            len(batch) if "batch" in locals() else "unknown"
+                        )
                         logger.error(
-                            f"[BUILD CACHE] Error details: batch size: "
-                            f"{len(batch) if 'batch' in locals() else 'unknown'}"
+                            f"[BUILD CACHE] Error details: batch size: {batch_len_info}"
                         )
+                        logger.exception(e)
 
             total_time = time.time() - start_time
             logger.info(
@@ -622,6 +648,7 @@ class LocalDuckDBStorageCache(BaseCacheManager):
                     time.sleep(self.extra_params.anti_quota_limit)
                 except Exception as err:
                     logger.error(f"Error in saving chunk: {str(err)}")
+                    logger.exception(err)
 
     def process_queue(self):
         while self.queue:
@@ -671,7 +698,8 @@ class LocalDuckDBStorageCache(BaseCacheManager):
                             self.failed_files.add(file_info.file_path)
                             save_failed_files(self.failed_files_path, self.failed_files)
                     except Exception as e:
-                        logger.error(f"Error in process_queue: {e}")
+                        logger.error(f"Error in process_queue: {str(e)}")
+                        logger.exception(e)
                         self.failed_files.add(file_info.file_path)
                         save_failed_files(self.failed_files_path, self.failed_files)
 
@@ -816,17 +844,18 @@ class LocalDuckDBStorageCache(BaseCacheManager):
                 for doc in cached_data.content:
                     if total_tokens + doc["tokens"] > self.max_output_tokens:
                         logger.info(
-                            f"当前检索已超出用户设置 Hybrid Index Max Tokens:{self.max_output_tokens}，"
-                            f"累计tokens: {total_tokens}, "
-                            f"经过向量搜索共检索出 {len(result.keys())} 个文档, 共 {len(self.cache.keys())} 个文档"
+                            f"当前检索已超出用户设置 Hybrid Index Max Tokens:"
+                            f"{self.max_output_tokens}，累计tokens: {total_tokens}, "
+                            f"经过向量搜索共检索出 {len(result.keys())} 个文档, "
+                            f"共 {len(self.cache.keys())} 个文档"
                         )
                         return result
                     total_tokens += doc["tokens"]
                 result[file_path] = cached_data.model_dump()
         logger.info(
             f"用户Hybrid Index Max Tokens设置为:{self.max_output_tokens}，"
-            f"累计tokens: {total_tokens}, "
-            f"经过向量搜索共检索出 {len(result.keys())} 个文档, 共 {len(self.cache.keys())} 个文档"
+            f"累计tokens: {total_tokens}, 经过向量搜索共检索出 "
+            f"{len(result.keys())} 个文档, 共 {len(self.cache.keys())} 个文档"
         )
         return result
 
@@ -904,6 +933,7 @@ class LocalDuckDBStorageCache(BaseCacheManager):
                     query_results.append((query, query_result))
                 except Exception as e:
                     logger.error(f"处理查询 '{query}' 时出错: {str(e)}")
+                    logger.exception(e)
 
         logger.info(f"所有查询共返回 {sum(len(r) for _, r in query_results)} 条记录")
 

{auto_coder-0.1.375 → auto_coder-0.1.376}/src/autocoder/rag/conversation_to_queries.py

@@ -2,7 +2,9 @@ from typing import List, Dict, Any, Optional, Union
 import logging
 import byzerllm
 from pydantic import BaseModel
+from autocoder.rag.types import RAGStat
 from autocoder.common import AutoCoderArgs
+from byzerllm import MetaHolder
 
 logger = logging.getLogger(__name__)
 
@@ -86,7 +88,7 @@ class ConversationToQueries:
         ```
         """
     
-    def extract_queries(self, conversations: List[Dict[str, Any]], max_queries: int = 3) -> List[SearchQuery]:
+    def extract_queries(self, conversations: List[Dict[str, Any]], max_queries: int = 3,rag_stat:Optional[RAGStat] = None) -> List[SearchQuery]:
         """
         从对话历史中提取搜索查询。
         
@@ -99,10 +101,39 @@ class ConversationToQueries:
         """
         try:
             # 使用 prompt 函数生成搜索查询
-            queries = self.generate_search_queries.with_llm(self.llm).with_return_type(SearchQuery).run(
+            model_name = self.llm.default_model_name
+            meta_holder = MetaHolder()
+            queries = self.generate_search_queries.with_llm(self.llm).with_return_type(SearchQuery).with_meta(
+                        meta_holder).run(
                 conversations=conversations,
                 max_queries=max_queries
-            )            
+            ) 
+
+            # 如果有元数据且有 rag_stat，则记录模型使用情况
+            if meta_holder.get_meta() and rag_stat:
+                meta_dict = meta_holder.get_meta()                    
+                input_tokens_count = meta_dict.get("input_tokens_count", 0) 
+                generated_tokens_count = meta_dict.get("generated_tokens_count", 0)
+                
+                # 检查模型是否已存在于 other_stats 中
+                found = False
+                for other_stat in rag_stat.other_stats:
+                    if other_stat.model_name == model_name:
+                        # 模型已存在，累加统计数据
+                        other_stat.total_input_tokens += input_tokens_count
+                        other_stat.total_generated_tokens += generated_tokens_count
+                        found = True
+                        break
+                
+                # 如果模型不存在，添加新的 OtherStat
+                if not found and (input_tokens_count > 0 or generated_tokens_count > 0):
+                    from autocoder.rag.types import OtherStat
+                    new_stat = OtherStat(
+                        total_input_tokens=input_tokens_count,
+                        total_generated_tokens=generated_tokens_count,
+                        model_name=model_name
+                    )
+                    rag_stat.other_stats.append(new_stat)
             
             # 按重要性排序
             queries.sort(key=lambda x: x.importance, reverse=True)
@@ -116,7 +147,8 @@ def extract_search_queries(
     conversations: List[Dict[str, Any]], 
     args:AutoCoderArgs,
     llm: Union[byzerllm.ByzerLLM, byzerllm.SimpleByzerLLM],
-    max_queries: int = 3,    
+    max_queries: int = 3,  
+    rag_stat:Optional[RAGStat] = None  
 ) -> List[SearchQuery]:
     """
     从对话历史中提取搜索查询的便捷函数。
@@ -133,7 +165,7 @@ def extract_search_queries(
         return []
     try:    
         extractor = ConversationToQueries(llm)
-        return extractor.extract_queries(conversations, max_queries) 
+        return extractor.extract_queries(conversations, max_queries,rag_stat) 
     except Exception as e:
         logger.error(f"Error extracting search queries from conversation: {str(e)}")
         return []