zrb 1.8.10__py3-none-any.whl → 1.21.29__py3-none-any.whl

zrb/builtin/llm/tool/code.py

@@ -3,39 +3,10 @@ import os
 
 from zrb.builtin.llm.tool.file import DEFAULT_EXCLUDED_PATTERNS, is_excluded
 from zrb.builtin.llm.tool.sub_agent import create_sub_agent_tool
+from zrb.config.config import CFG
+from zrb.config.llm_rate_limitter import llm_rate_limitter
 from zrb.context.any_context import AnyContext
-from zrb.llm_rate_limitter import llm_rate_limitter
-
-_EXTRACT_INFO_FROM_REPO_SYSTEM_PROMPT = """
-You are an extraction info agent.
-Your goal is to help to extract relevant information to help the main assistant.
-You write your output is in markdown format containing path and relevant information.
-Extract only information that relevant to main assistant's goal.
-
-Extracted Information format (Use this as reference, extract relevant information only):
-# <file-name>
-## imports
-- <imported-package>
-- ...
-## variables
-- <variable-type> <variable-name>: <the-purpose-of-the-variable>
-- ...
-## functions
-- <function-name>:
-  - parameters: <parameters>
-  - logic/description: <what-the-function-do-and-how-it-works>
-...
-# <other-file-name>
-...
-""".strip()
-
-
-_SUMMARIZE_INFO_SYSTEM_PROMPT = """
-You are an information summarization agent.
-Your goal is to summarize information to help the main assistant.
-The summarization result should contains all necessary details
-to help main assistant achieve the goal.
-"""
+from zrb.util.cli.style import stylize_faint
 
 _DEFAULT_EXTENSIONS = [
     "py",
@@ -79,58 +50,67 @@ _DEFAULT_EXTENSIONS = [
 async def analyze_repo(
     ctx: AnyContext,
     path: str,
-    goal: str,
+    query: str,
     extensions: list[str] = _DEFAULT_EXTENSIONS,
     exclude_patterns: list[str] = DEFAULT_EXCLUDED_PATTERNS,
-    extraction_token_limit: int = 30000,
-    summarization_token_limit: int = 30000,
+    extraction_token_threshold: int | None = None,
+    summarization_token_threshold: int | None = None,
 ) -> str:
     """
-    Extract and summarize information from a directory that probably
-    contains a large resources.
-    You should state the goal specifically so that the tool can return relevant informations.
-    Use this tool for:
-    - summarization
-    - outline/structure extraction
-    - code review
-    - create diagram as code
-    - other tasks
+    Analyzes a code repository or directory to answer a specific query.
+
+    CRITICAL: The quality of analysis depends entirely on the query. Vague queries yield poor
+        results.
+    IMPORTANT: This tool can be slow and expensive on large repositories. Use judiciously.
+
+    Example:
+    analyze_repo(
+        path='src/my_project',
+        query='Summarize the main functionalities by analyzing Python files.',
+        extensions=['py']
+    )
+
     Args:
-        path (str): File path to be analyze. Pass exactly as provided, including '~'.
-        goal(str): Goal of extracting information (for example creating C4 diagram)
-        extensions(Optional[list[str]]): List of extension to be included
-            while reading resources. Defaults to common programming languages and config files.
-        exclude_patterns(Optional[list[str]]): List of patterns to exclude from analysis.
-            Common patterns like '.venv', 'node_modules' should be excluded by default.
-        extraction_token_limit(Optional[int]): Max resource content char length
-            the extraction assistant able to handle. Defaults to 150000
-        summarization_token_limit(Optional[int]): Max resource content char length
-            the summarization assistant able to handle. Defaults to 150000
+        ctx (AnyContext): The execution context.
+        path (str): Path to the directory or repository.
+        query (str): Clear and specific analysis question or goal.
+        extensions (list[str], optional): File extensions to include.
+        exclude_patterns (list[str], optional): Glob patterns to exclude.
+        extraction_token_threshold (int, optional): Token limit for extraction sub-agent.
+        summarization_token_threshold (int, optional): Token limit for summarization sub-agent.
+
     Returns:
-        str: The analysis result
-    Raises:
-        Exception: If an error occurs.
+        str: Detailed, markdown-formatted analysis and summary.
     """
+    if extraction_token_threshold is None:
+        extraction_token_threshold = CFG.LLM_REPO_ANALYSIS_EXTRACTION_TOKEN_THRESHOLD
+    if summarization_token_threshold is None:
+        summarization_token_threshold = (
+            CFG.LLM_REPO_ANALYSIS_SUMMARIZATION_TOKEN_THRESHOLD
+        )
     abs_path = os.path.abspath(os.path.expanduser(path))
     file_metadatas = _get_file_metadatas(abs_path, extensions, exclude_patterns)
-    ctx.print("Extraction")
+    ctx.print(stylize_faint("  📝 Extraction"), plain=True)
     extracted_infos = await _extract_info(
         ctx,
         file_metadatas=file_metadatas,
-        goal=goal,
-        token_limit=extraction_token_limit,
+        query=query,
+        token_limit=extraction_token_threshold,
     )
+    if len(extracted_infos) == 0:
+        raise RuntimeError(
+            "No info can be extracted, adjust extensions or exclude_patterns."
+        )
     if len(extracted_infos) == 1:
         return extracted_infos[0]
-    ctx.print("Summarization")
     summarized_infos = extracted_infos
     while len(summarized_infos) > 1:
-        ctx.print("Summarization")
+        ctx.print(stylize_faint("  📝 Summarization"), plain=True)
         summarized_infos = await _summarize_info(
             ctx,
             extracted_infos=summarized_infos,
-            goal=goal,
-            token_limit=summarization_token_limit,
+            query=query,
+            token_limit=summarization_token_threshold,
         )
     return summarized_infos[0]
 
@@ -147,11 +127,11 @@ def _get_file_metadatas(
             if not any(file.endswith(f".{ext}") for ext in extensions):
                 continue
             file_path = os.path.join(root, file)
-            if is_excluded(file_path, exclude_patterns):
-                continue
             try:
+                rel_path = os.path.relpath(file_path, dir_path)
+                if is_excluded(rel_path, exclude_patterns):
+                    continue
                 with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
-                    rel_path = os.path.relpath(file_path, dir_path)
                     metadata_list.append({"path": rel_path, "content": f.read()})
             except Exception as e:
                 print(f"Error reading file {file_path}: {e}")
@@ -162,13 +142,15 @@ def _get_file_metadatas(
 async def _extract_info(
     ctx: AnyContext,
     file_metadatas: list[dict[str, str]],
-    goal: str,
+    query: str,
     token_limit: int,
 ) -> list[str]:
     extract = create_sub_agent_tool(
         tool_name="extract",
         tool_description="extract",
-        system_prompt=_EXTRACT_INFO_FROM_REPO_SYSTEM_PROMPT,
+        system_prompt=CFG.LLM_REPO_EXTRACTOR_SYSTEM_PROMPT,
+        auto_summarize=False,
+        remember_history=False,
     )
     extracted_infos = []
     content_buffer = []
@@ -180,7 +162,7 @@ async def _extract_info(
         file_str = json.dumps(file_obj)
         if current_token_count + llm_rate_limitter.count_token(file_str) > token_limit:
             if content_buffer:
-                prompt = _create_extract_info_prompt(goal, content_buffer)
+                prompt = json.dumps(_create_extract_info_prompt(query, content_buffer))
                 extracted_info = await extract(
                     ctx, llm_rate_limitter.clip_prompt(prompt, token_limit)
                 )
@@ -190,10 +172,9 @@ async def _extract_info(
         else:
             content_buffer.append(file_obj)
             current_token_count += llm_rate_limitter.count_token(file_str)
-
     # Process any remaining content in the buffer
     if content_buffer:
-        prompt = _create_extract_info_prompt(goal, content_buffer)
+        prompt = json.dumps(_create_extract_info_prompt(query, content_buffer))
         extracted_info = await extract(
             ctx, llm_rate_limitter.clip_prompt(prompt, token_limit)
         )
@@ -201,25 +182,25 @@ async def _extract_info(
     return extracted_infos
 
 
-def _create_extract_info_prompt(goal: str, content_buffer: list[dict]) -> str:
-    return json.dumps(
-        {
-            "main_assistant_goal": goal,
-            "files": content_buffer,
-        }
-    )
+def _create_extract_info_prompt(query: str, content_buffer: list[dict]) -> dict:
+    return {
+        "main_assistant_query": query,
+        "files": content_buffer,
+    }
 
 
 async def _summarize_info(
     ctx: AnyContext,
     extracted_infos: list[str],
-    goal: str,
+    query: str,
     token_limit: int,
 ) -> list[str]:
     summarize = create_sub_agent_tool(
         tool_name="extract",
         tool_description="extract",
-        system_prompt=_SUMMARIZE_INFO_SYSTEM_PROMPT,
+        system_prompt=CFG.LLM_REPO_SUMMARIZER_SYSTEM_PROMPT,
+        auto_summarize=False,
+        remember_history=False,
     )
     summarized_infos = []
     content_buffer = ""
@@ -227,7 +208,9 @@ async def _summarize_info(
         new_prompt = content_buffer + extracted_info
         if llm_rate_limitter.count_token(new_prompt) > token_limit:
             if content_buffer:
-                prompt = _create_summarize_info_prompt(goal, content_buffer)
+                prompt = json.dumps(
+                    _create_summarize_info_prompt(query, content_buffer)
+                )
                 summarized_info = await summarize(
                     ctx, llm_rate_limitter.clip_prompt(prompt, token_limit)
                 )
@@ -238,7 +221,7 @@ async def _summarize_info(
 
     # Process any remaining content in the buffer
     if content_buffer:
-        prompt = _create_summarize_info_prompt(goal, content_buffer)
+        prompt = json.dumps(_create_summarize_info_prompt(query, content_buffer))
         summarized_info = await summarize(
             ctx, llm_rate_limitter.clip_prompt(prompt, token_limit)
         )
@@ -246,10 +229,8 @@ async def _summarize_info(
     return summarized_infos
 
 
-def _create_summarize_info_prompt(goal: str, content_buffer: str) -> str:
-    return json.dumps(
-        {
-            "main_assistant_goal": goal,
-            "extracted_info": content_buffer,
-        }
-    )
+def _create_summarize_info_prompt(query: str, content_buffer: str) -> dict:
+    return {
+        "main_assistant_query": query,
+        "extracted_info": content_buffer,
+    }