auto-coder 0.1.205__py3-none-any.whl → 0.1.206__py3-none-any.whl

autocoder/benchmark.py

@@ -0,0 +1,135 @@
+from openai import AsyncOpenAI
+import asyncio
+import time
+from rich.console import Console
+from rich.table import Table
+import numpy as np
+import ray
+from loguru import logger
+import byzerllm
+from concurrent.futures import ThreadPoolExecutor
+
+async def benchmark_openai(model: str, parallel: int, api_key: str, base_url: str = None, rounds: int = 1):
+    client = AsyncOpenAI(api_key=api_key, base_url=base_url if base_url else None)
+    start_time = time.time()
+    
+    async def single_request():
+        try:
+            t1 = time.time()
+            response = await client.chat.completions.create(
+                model=model,
+                messages=[{"role": "user", "content": "Hello, how are you?"}]
+            )
+            t2 = time.time()
+            return t2 - t1
+        except Exception as e:
+            logger.error(f"Request failed: {e}")
+            return None
+
+    all_results = []
+    for round_num in range(rounds):
+        print(f"Running round {round_num + 1}/{rounds}")
+        tasks = [single_request() for _ in range(parallel)]
+        results = await asyncio.gather(*tasks)
+        all_results.extend(results)
+    
+    results = all_results
+    
+    # Filter out None values from failed requests
+    results = [r for r in results if r is not None]
+    
+    end_time = time.time()
+    total_time = end_time - start_time
+    
+    if not results:
+        print("All requests failed")
+        return
+
+    # Calculate statistics
+    avg_time = np.mean(results)
+    p50 = np.percentile(results, 50)
+    p90 = np.percentile(results, 90)
+    p95 = np.percentile(results, 95)
+    p99 = np.percentile(results, 99)
+    
+    # Create rich table for output
+    console = Console()
+    table = Table(title=f"OpenAI Client Benchmark Results (Parallel={parallel})")
+    
+    table.add_column("Metric", style="cyan")
+    table.add_column("Value (seconds)", style="magenta")
+    
+    table.add_row("Total Time", f"{total_time:.2f}")
+    table.add_row("Average Response Time", f"{avg_time:.2f}")
+    table.add_row("Median (P50)", f"{p50:.2f}")
+    table.add_row("P90", f"{p90:.2f}")
+    table.add_row("P95", f"{p95:.2f}")
+    table.add_row("P99", f"{p99:.2f}")
+    table.add_row("Requests/Second", f"{parallel/total_time:.2f}")
+    
+    console.print(table)
+
+def benchmark_byzerllm(model: str, parallel: int, rounds: int = 1):
+    byzerllm.connect_cluster(address="auto")
+    llm = byzerllm.ByzerLLM()
+    llm.setup_default_model_name(model)    
+
+    def single_request(llm):
+        try:
+            t1 = time.time()
+            llm.chat_oai(conversations=[{
+                "role": "user",
+                "content": "Hello, how are you?"
+            }])
+            t2 = time.time()
+            return t2 - t1
+        except Exception as e:
+            logger.error(f"Request failed: {e}")
+            return None
+
+    start_time = time.time()
+    all_results = []
+    for round_num in range(rounds):
+        print(f"Running round {round_num + 1}/{rounds}")
+        with ThreadPoolExecutor(max_workers=parallel) as executor:
+            # submit tasks to the executor
+            futures = [executor.submit(single_request, llm) for _ in range(parallel)]
+            # get results from futures
+            results = [future.result() for future in futures]
+            all_results.extend(results)
+    
+        results = all_results
+        
+        # Filter out None values from failed requests
+        results = [r for r in results if r is not None]
+        
+        end_time = time.time()
+        total_time = end_time - start_time
+
+        if not results:
+            print("All requests failed")
+            return
+
+        # Calculate statistics
+        avg_time = np.mean(results)
+        p50 = np.percentile(results, 50)
+        p90 = np.percentile(results, 90)
+        p95 = np.percentile(results, 95)
+        p99 = np.percentile(results, 99)
+        
+        # Create rich table for output
+        console = Console()
+        table = Table(title=f"ByzerLLM Client Benchmark Results (Parallel={parallel})")
+        
+        table.add_column("Metric", style="cyan")
+        table.add_column("Value (seconds)", style="magenta")
+        
+        table.add_row("Total Time", f"{total_time:.2f}")
+        table.add_row("Average Response Time", f"{avg_time:.2f}")
+        table.add_row("Median (P50)", f"{p50:.2f}")
+        table.add_row("P90", f"{p90:.2f}")
+        table.add_row("P95", f"{p95:.2f}")
+        table.add_row("P99", f"{p99:.2f}")
+        table.add_row("Requests/Second", f"{parallel/total_time:.2f}")
+        
+        console.print(table)

autocoder/chat_auto_coder.py

@@ -58,6 +58,7 @@ from prompt_toolkit.patch_stdout import patch_stdout
 import byzerllm
 from byzerllm.utils import format_str_jinja2
 from autocoder.chat_auto_coder_lang import get_message
+from autocoder.utils import operate_config_api
 
 
 class SymbolItem(BaseModel):

autocoder/common/chunk_validation.py

@@ -0,0 +1,91 @@
+from typing import List, Optional
+import byzerllm
+from loguru import logger
+import json
+from byzerllm.utils.client.code_utils import extract_code
+
+## This function is generated by auto-coder.chat in Korea
+def validate_chunk(llm: byzerllm.ByzerLLM, content: Optional[List[str]] = None, query: Optional[str] = None) -> str:
+    """
+    验证文本分块模型的效果
+    
+    Args:
+        llm: ByzerLLM实例
+        content: 待验证的内容列表
+        query: 相关问题
+    
+    Returns:
+        验证结果
+    """
+    if content is None:
+        content = [
+            """
+class TokenLimiter:
+    def __init__(
+        self,
+        count_tokens: Callable[[str], int],
+        full_text_limit: int,
+        segment_limit: int,
+        buff_limit: int,
+        llm:ByzerLLM,
+        disable_segment_reorder: bool,
+    ):
+        self.count_tokens = count_tokens
+        self.full_text_limit = full_text_limit
+        self.segment_limit = segment_limit
+        self.buff_limit = buff_limit
+        self.llm = llm
+
+    def limit_tokens(self, relevant_docs: List[SourceCode]):
+        pass
+            """
+        ]
+    
+    if query is None:
+        query = "What are the main methods in TokenLimiter class?"
+
+    try:
+        from autocoder.rag.token_limiter import TokenLimiter
+        def count_tokens(text:str):
+            return 0
+        token_limiter = TokenLimiter(
+            llm=llm,
+            count_tokens=count_tokens,
+            full_text_limit=1000,
+            segment_limit=1000,
+            buff_limit=1000,
+            disable_segment_reorder=False
+        )
+        conversations = [
+            {"role": "user", "content": query}
+        ]
+        result = token_limiter.extract_relevance_range_from_docs_with_conversation.with_llm(llm).run(conversations, content)
+        
+        # 结果验证和解析
+        validation_result = []
+        for doc_idx, doc in enumerate(content):
+            doc_lines = doc.split('\n')
+            source_code_with_line_number = ""
+            for idx, line in enumerate(doc_lines):
+                source_code_with_line_number += f"{idx+1} {line}\n"
+            
+            json_str = extract_code(result)[0][1]
+            json_objs = json.loads(json_str)
+            
+            for json_obj in json_objs:
+                start_line = json_obj["start_line"] - 1
+                end_line = json_obj["end_line"]
+                if start_line >= 0 and end_line > start_line and end_line <= len(doc_lines):
+                    chunk = "\n".join(doc_lines[start_line:end_line])
+                    validation_result.append(
+                        f"Document {doc_idx + 1} - Extracted Range (lines {json_obj['start_line']}-{json_obj['end_line']}):\n{chunk}"
+                    )
+
+        if not validation_result:
+            return "No valid ranges extracted from the documents."
+
+        return "\n\n".join(validation_result)
+
+    except Exception as e:
+        logger.error(f"Error validating chunk model: {str(e)}")
+        return f"Error: {str(e)}"

autocoder/common/recall_validation.py

@@ -0,0 +1,58 @@
+from typing import List, Optional
+import byzerllm
+from loguru import logger
+from autocoder.rag.doc_filter import _check_relevance_with_conversation
+from autocoder.rag.relevant_utils import parse_relevance
+
+def validate_recall(llm: byzerllm.ByzerLLM, content: Optional[List[str]] = None, query: Optional[str] = None) -> bool:
+    """
+    验证召回模型的效果
+    
+    Args:
+        llm: ByzerLLM实例
+        content: 待验证的内容列表
+        query: 查询语句
+    
+    Returns:
+        验证成功返回True，失败返回False
+    """
+    if content is None:
+        content = [
+            """
+            # ByzerLLM API Guide
+            
+            ByzerLLM provides a simple API for interacting with language models. 
+            Here's how to use it:
+            
+            1. Initialize the client
+            2. Send requests
+            3. Process responses
+            
+            Example:
+            ```python
+            import byzerllm
+            llm = byzerllm.ByzerLLM()
+            response = llm.chat(prompt="Hello")
+            ```
+            """
+        ]
+    
+    if query is None:
+        query = "How do I use the ByzerLLM API?"
+    
+    conversations = [
+        {"role": "user", "content": query}
+    ]
+    
+    try:
+        relevance_str = _check_relevance_with_conversation.with_llm(llm).run(conversations, content)
+        relevance = parse_relevance(relevance_str)
+        
+        if relevance is None:
+            logger.error("Failed to parse relevance result")
+            return False
+            
+        return relevance.is_relevant
+    except Exception as e:
+        logger.error(f"Error validating recall: {str(e)}")
+        return False