auto-coder 0.1.297__tar.gz → 0.1.298__tar.gz

{auto_coder-0.1.297 → auto_coder-0.1.298}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.297
+Version: 0.1.298
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -26,7 +26,8 @@ Requires-Dist: tabulate
 Requires-Dist: jupyter_client
 Requires-Dist: prompt-toolkit
 Requires-Dist: tokenizers
-Requires-Dist: byzerllm[saas]>=0.1.174
+Requires-Dist: aiofiles
+Requires-Dist: byzerllm[saas]>=0.1.176
 Requires-Dist: patch
 Requires-Dist: diff_match_patch
 Requires-Dist: GitPython

{auto_coder-0.1.297 → auto_coder-0.1.298}/src/auto_coder.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: auto-coder
-Version: 0.1.297
+Version: 0.1.298
 Summary: AutoCoder: AutoCoder
 Author: allwefantasy
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -26,7 +26,8 @@ Requires-Dist: tabulate
 Requires-Dist: jupyter_client
 Requires-Dist: prompt-toolkit
 Requires-Dist: tokenizers
-Requires-Dist: byzerllm[saas]>=0.1.174
+Requires-Dist: aiofiles
+Requires-Dist: byzerllm[saas]>=0.1.176
 Requires-Dist: patch
 Requires-Dist: diff_match_patch
 Requires-Dist: GitPython

{auto_coder-0.1.297 → auto_coder-0.1.298}/src/auto_coder.egg-info/requires.txt

@@ -16,7 +16,8 @@ tabulate
 jupyter_client
 prompt-toolkit
 tokenizers
-byzerllm[saas]>=0.1.174
+aiofiles
+byzerllm[saas]>=0.1.176
 patch
 diff_match_patch
 GitPython

{auto_coder-0.1.297 → auto_coder-0.1.298}/src/autocoder/auto_coder_rag.py

@@ -187,6 +187,13 @@ def main(input_args: Optional[List[str]] = None):
         help="The storage type of the RAG, duckdb or byzer-storage",
     )
 
+    build_index_parser.add_argument(
+        "--rag_index_build_workers",
+        type=int,
+        default=5,
+        help="The number of workers to build the RAG index",
+    )
+
     build_index_parser.add_argument(
         "--quick", action="store_true", help="Skip system initialization"
     )

{auto_coder-0.1.297 → auto_coder-0.1.298}/src/autocoder/common/__init__.py

@@ -300,6 +300,7 @@ class AutoCoderArgs(pydantic.BaseModel):
     rag_duckdb_vector_dim: Optional[int] = 1024  # DuckDB 向量化存储的维度
     rag_duckdb_query_similarity: Optional[float] = 0.1  # DuckDB 向量化检索 相似度 阈值
     rag_duckdb_query_top_k: Optional[int] = 10000  # DuckDB 向量化检索 返回 TopK个结果(且大于相似度)
+    rag_index_build_workers: Optional[int] = 10
     # rag 本地图床地址
     local_image_host: Optional[str] = ""
     rag_recall_max_queries: Optional[int] = 5

{auto_coder-0.1.297 → auto_coder-0.1.298}/src/autocoder/rag/cache/byzer_storage_cache.py

@@ -132,6 +132,7 @@ class ByzerStorageCache(BaseCacheManager):
         self.path = path
         self.ignore_spec = ignore_spec
         self.required_exts = required_exts
+        self.extra_params = extra_params
         self.rag_build_name = extra_params.rag_build_name
         self.storage = ByzerStorage("byzerai_store", "rag", self.rag_build_name)
         self.queue = []
@@ -329,7 +330,7 @@ class ByzerStorageCache(BaseCacheManager):
             start_time = time.time()
             
             # Use more workers to process the smaller batches efficiently
-            max_workers = min(10, total_batches)  # Cap at 10 workers or total batch count
+            max_workers = min(self.extra_params.rag_index_build_workers, total_batches)  # Cap at 10 workers or total batch count
             logger.info(f"[BUILD CACHE] Using {max_workers} parallel workers for processing")
             
             with ThreadPoolExecutor(max_workers=max_workers) as executor:

{auto_coder-0.1.297 → auto_coder-0.1.298}/src/autocoder/rag/cache/local_byzer_storage_cache.py

@@ -338,7 +338,7 @@ class LocalByzerStorageCache(BaseCacheManager):
             start_time = time.time()
 
             # Use more workers to process the smaller batches efficiently
-            max_workers = min(10, total_batches)  # Cap at 10 workers or total batch count
+            max_workers = min(self.extra_params.rag_index_build_workers, total_batches)  # Cap at 10 workers or total batch count
             logger.info(f"[BUILD CACHE] Using {max_workers} parallel workers for processing")
 
             with ThreadPoolExecutor(max_workers=max_workers) as executor:

{auto_coder-0.1.297 → auto_coder-0.1.298}/src/autocoder/rag/cache/local_duckdb_storage_cache.py

@@ -133,15 +133,31 @@ class LocalDuckdbStorage:
         return reduced
 
     def _embedding(self, context: str, norm: bool = True, dim: int | None = None) -> List[float]:
-        embedding = self.llm.emb_query(context)[0].output
-
-        if dim:
-            embedding = self._apply_pca(embedding, target_dim=dim)  # 降维后形状 (1024,)
-
-        if norm:
-            embedding = embedding / np.linalg.norm(embedding)
-
-        return embedding.tolist()
+        max_retries = 3
+        retry_count = 0
+        
+        while retry_count < max_retries:
+            try:
+                embedding = self.llm.emb_query(context)[0].output
+                
+                if dim:
+                    embedding = self._apply_pca(embedding, target_dim=dim)  # 降维后形状 (1024,)
+                
+                if norm:
+                    embedding = embedding / np.linalg.norm(embedding)
+                
+                return embedding.tolist()
+            except Exception as e:
+                retry_count += 1
+                if retry_count >= max_retries:
+                    logger.error(f"Failed to get embedding after {max_retries} attempts: {str(e)}")
+                    raise
+                
+                # Sleep between 1-5 seconds before retrying
+                sleep_time = 1 + (retry_count * 1.5)
+                logger.warning(f"Embedding API call failed (attempt {retry_count}/{max_retries}). "
+                              f"Error: {str(e)}. Retrying in {sleep_time:.1f} seconds...")
+                time.sleep(sleep_time)
 
     def _initialize(self) -> None:
         if self.embed_dim is None:
@@ -452,9 +468,9 @@ class LocalDuckDBStorageCache(BaseCacheManager):
         self.write_cache()
 
         if items:
-            logger.info("[BUILD CACHE] Clearing existing cache from Byzer DuckDB Storage")
+            logger.info("[BUILD CACHE] Clearing existing cache from DuckDB  Storage")
             self.storage.truncate_table()
-            logger.info(f"[BUILD CACHE] Preparing to write to Byzer DuckDB Storage, "
+            logger.info(f"[BUILD CACHE] Preparing to write to DuckDB  Storage, "
                         f"total chunks: {len(items)}, total files: {len(files_to_process)}")
 
             # Use a fixed optimal batch size instead of dividing by worker count
@@ -464,12 +480,12 @@ class LocalDuckDBStorageCache(BaseCacheManager):
             total_batches = len(item_batches)
             completed_batches = 0
 
-            logger.info(f"[BUILD CACHE] Starting to write to Byzer Storage using {batch_size} items per batch, "
+            logger.info(f"[BUILD CACHE] Starting to write to DuckDB Storage using {batch_size} items per batch, "
                         f"total batches: {total_batches}")
             start_time = time.time()
 
-            # Use more workers to process the smaller batches efficiently
-            max_workers = min(10, total_batches)  # Cap at 10 workers or total batch count
+            # Use more workers to process the smaller batches efficiently            
+            max_workers = min(self.extra_params.rag_index_build_workers, total_batches)  # Cap at 10 workers or total batch count
             logger.info(f"[BUILD CACHE] Using {max_workers} parallel workers for processing")
 
             def batch_add_doc(_batch):

auto_coder-0.1.298/src/autocoder/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.298"

auto_coder-0.1.297/src/autocoder/version.py

@@ -1 +0,0 @@
-__version__ = "0.1.297"