bioguider 0.2.31__py3-none-any.whl → 0.2.33__py3-none-any.whl

bioguider/agents/agent_utils.py

@@ -27,6 +27,7 @@ from bioguider.utils.utils import clean_action_input
 from ..utils.gitignore_checker import GitignoreChecker
 from ..database.summarized_file_db import SummarizedFilesDb
 from bioguider.agents.common_conversation import CommonConversation
+from bioguider.rag.config import configs
 
 logger = logging.getLogger(__name__)
 
@@ -66,8 +67,17 @@ def get_llm(
     api_version: str=None,
     azure_deployment: str=None,
     temperature: float = 0.0,
-    max_tokens: int = 4096,
+    max_tokens: int = 16384,  # Set high by default - enough for any document type
 ):
+    """
+    Create an LLM instance with appropriate parameters based on model type and API version.
+    
+    Handles parameter compatibility across different models and API versions:
+    - DeepSeek models: Use max_tokens parameter
+    - GPT models (newer): Use max_completion_tokens parameter
+    - GPT-5+: Don't support custom temperature (uses default)
+    """
+    
     if model_name.startswith("deepseek"):
         chat = ChatDeepSeek(
             api_key=api_key,
@@ -76,23 +86,38 @@ def get_llm(
             max_tokens=max_tokens,
         )
     elif model_name.startswith("gpt"):
-        chat = AzureChatOpenAI(
-            api_key=api_key,
-            azure_endpoint=azure_endpoint,
-            api_version=api_version,
-            azure_deployment=azure_deployment,
-            model=model_name,
-            temperature=temperature,
-            max_tokens=max_tokens,
-        )
+        # Base parameters common to all GPT models
+        llm_params = {
+            "api_key": api_key,
+            "azure_endpoint": azure_endpoint,
+            "api_version": api_version,
+            "azure_deployment": azure_deployment,
+            "model": model_name,
+        }
+        
+        # Determine token limit parameter name based on API version
+        # Newer APIs (2024-08+) use max_completion_tokens instead of max_tokens
+        use_completion_tokens = api_version and api_version >= "2024-08-01-preview"
+        token_param = "max_completion_tokens" if use_completion_tokens else "max_tokens"
+        llm_params[token_param] = max_tokens
+        
+        # Handle temperature parameter based on model capabilities
+        # GPT-5+ models don't support custom temperature values
+        supports_temperature = not any(restricted in model_name for restricted in ["gpt-5", "o1", "o3"])
+        if supports_temperature:
+            llm_params["temperature"] = temperature
+            
+        chat = AzureChatOpenAI(**llm_params)
     else:
-        raise ValueError("Invalid model name")
-    # validate chat
+        raise ValueError(f"Unsupported model type: {model_name}")
+    
+    # Validate the LLM instance with a simple test
     try:
         chat.invoke("Hi")
     except Exception as e:
-        print(e)
+        logger.error(f"Failed to initialize LLM {model_name}: {e}")
         return None
+    
     return chat
 
 def pretty_print(message, printout = True):
@@ -153,7 +178,9 @@ def read_directory(
         return None
     gitignore_checker = GitignoreChecker(
         directory=dir_path,
-        gitignore_path=gitignore_path
+        gitignore_path=gitignore_path,
+        exclude_dir_patterns=configs["file_filters"]["excluded_dirs"],
+        exclude_file_patterns=configs["file_filters"]["excluded_files"],
     )
     files = gitignore_checker.check_files_and_folders(level=level)
     return files

bioguider/agents/evaluation_readme_task.py

@@ -28,6 +28,7 @@ from bioguider.utils.constants import (
     EvaluationREADMEResult,
 )
 from bioguider.utils.utils import increase_token_usage
+from bioguider.rag.config import configs
 
 logger = logging.getLogger(__name__)
 
@@ -638,7 +639,9 @@ class EvaluationREADMETask(EvaluationTask):
         repo_path = self.repo_path
         gitignore_path = Path(repo_path, ".gitignore")
         gitignore_checker = GitignoreChecker(
-            directory=repo_path, gitignore_path=gitignore_path
+            directory=repo_path, gitignore_path=gitignore_path,
+            exclude_dir_patterns=configs["file_filters"]["excluded_dirs"],
+            exclude_file_patterns=configs["file_filters"]["excluded_files"],
         )
         found_readme_files = gitignore_checker.check_files_and_folders(
             check_file_cb=lambda root_dir, relative_path: Path(relative_path).name.lower() in possible_readme_files,

bioguider/generation/llm_content_generator.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from typing import Dict
 import json
+import re
 from langchain_openai.chat_models.base import BaseChatOpenAI
 
 from bioguider.agents.common_conversation import CommonConversation
@@ -25,13 +26,14 @@ CRITICAL REQUIREMENTS
 - Follow the guidance EXACTLY as provided: {guidance}
 - Address the specific suggestions from the evaluation report precisely
 - Do not deviate from the guidance or add unrelated content
-- If guidance mentions specific packages, requirements, or details, include them exactly
-- For RMarkdown files (.Rmd), preserve the original structure including YAML frontmatter, code chunks, and existing headers
+- If guidance mentions specific packages, requirements, or details, include them ONLY if they are explicitly stated - never invent or estimate
+- Preserve the original file structure including frontmatter, code blocks, and existing headers
 - NEVER generate generic placeholder content like "Clear 2–3 sentence summary" or "brief description"
+- NEVER invent technical specifications (hardware requirements, version numbers, performance metrics) unless explicitly provided in guidance or context
 - ABSOLUTELY FORBIDDEN: Do NOT add summary sections, notes, conclusions, or any text at the end of documents
 - ABSOLUTELY FORBIDDEN: Do NOT wrap content in markdown code fences (```markdown). Return pure content only.
 - ABSOLUTELY FORBIDDEN: Do NOT add phrases like "Happy analyzing!", "Ensure all dependencies are up-to-date", or any concluding statements
-- ALWAYS use the specific guidance provided above to create concrete, actionable content
+- ALWAYS use the specific guidance provided above to create concrete, actionable content based on evidence
 
 STYLE & CONSTRAINTS
 - Fix obvious errors in the content.
@@ -47,22 +49,22 @@ STYLE & CONSTRAINTS
 - When targeting README content, do not rewrite the document title or header area; generate only the requested section body to be inserted below existing headers/badges.
 
 SECTION GUIDELINES (follow guidance exactly)
-- Dependencies: Include specific packages mentioned in guidance (e.g., "ggplot2", "dplyr", etc.)
-- System Requirements: Include R version requirements and platform-specific instructions as mentioned in guidance
-- Hardware Requirements: Include RAM/CPU recommendations as specified in guidance
+- Dependencies: Include ONLY specific packages explicitly mentioned in guidance or found in repo context. Never invent package names or versions.
+- System Requirements: Include ONLY language/runtime version requirements explicitly stated in guidance or found in repo context. Never invent version numbers.
+- Hardware Requirements: Include ONLY specific RAM/CPU recommendations explicitly stated in guidance or found in repo context. NEVER estimate or invent hardware specifications - omit this section if not substantiated.
 - License: one sentence referencing the license and pointing to the LICENSE file.
-- Install (clarify dependencies): Include compatibility details across operating systems and architectures as mentioned in guidance
+- Install (clarify dependencies): Include compatibility details ONLY if explicitly mentioned in guidance or found in repo context.
 - Tutorial improvements: Add specific examples, error handling, and reproducibility notes as mentioned in guidance
 - User guide improvements: Enhance clarity, add missing information, and improve error handling as mentioned in guidance
-- Conservative injection: For tutorial files (.Rmd), make minimal, targeted additions that preserve the original structure and flow. Add brief notes, small subsections, or contextual comments that enhance existing content without disrupting the tutorial's narrative.
-- RMarkdown integration: When inserting content into existing RMarkdown tutorials, integrate naturally into the flow rather than creating standalone sections. Add brief explanatory text, code comments, or small subsections that enhance the existing content.
-- RMarkdown format compliance: For .Rmd files, ensure content follows RMarkdown conventions:
-  * Use proper R code chunks with ```{{r chunk_name}} and ``` when adding code examples
+- Conservative injection: For tutorial files, make minimal, targeted additions that preserve the original structure and flow. Add brief notes, small subsections, or contextual comments that enhance existing content without disrupting the tutorial's narrative.
+- Natural integration: When inserting content into existing tutorials or guides, integrate naturally into the flow rather than creating standalone sections. Add brief explanatory text, code comments, or small subsections that enhance the existing content.
+- Format compliance: Preserve the existing file format conventions (e.g., YAML frontmatter, code blocks, headers):
+  * For code examples, use the appropriate code fence syntax for the language (e.g., ```r, ```python, ```bash)
   * Maintain the tutorial's existing tone and context - content should feel like a natural continuation
   * Avoid creating new major sections unless absolutely necessary
-  * Use inline R code with `{{r code_here}}` when appropriate
   * Keep explanations concise and contextual to the tutorial's purpose
-- Context awareness: Content should feel like a natural part of the existing tutorial, not a standalone addition. Reference the tutorial's specific context, datasets, and examples.
+- Context awareness: Content should feel like a natural part of the existing document, not a standalone addition. Reference the document's specific context, datasets, and examples when available.
+- Biological accuracy: For biomedical/bioinformatics content, ensure technical accuracy. If unsure about biological or computational details, keep descriptions general rather than inventing specifics.
 - If the section does not fit the above, produce content that directly addresses the guidance provided.
 
 OUTPUT FORMAT
@@ -74,40 +76,72 @@ OUTPUT FORMAT
 """
 
 LLM_FULLDOC_PROMPT = """
-You are "BioGuider," a documentation rewriter.
+You are "BioGuider," a documentation rewriter with enhanced capabilities for complex documents.
 
 GOAL
-Rewrite a complete target document using only the provided evaluation report signals and the repository context excerpts. Output a full, ready-to-publish markdown file that is more complete and directly usable.
+Rewrite a complete target document using only the provided evaluation report signals and the repository context excerpts. Output a full, ready-to-publish markdown file that is more complete and directly usable. You now have increased token capacity to handle complex documents comprehensively.
 
 INPUTS (authoritative)
 - evaluation_report (structured JSON excerpts): <<{evaluation_report}>>
 - target_file: {target_file}
 - repo_context_excerpt (do not copy blindly; use only to keep style/tone): <<{context}>>
 
+CRITICAL: SINGLE DOCUMENT WITH MULTIPLE IMPROVEMENTS
+This file requires improvements from {total_suggestions} separate evaluation suggestions. You must:
+1. **Read ALL {total_suggestions} suggestions** in the evaluation_report before writing
+2. **Integrate ALL suggestions into ONE cohesive document** - do NOT create {total_suggestions} separate versions
+3. **Weave improvements together naturally** - related suggestions should enhance the same sections
+4. **Write the document ONCE** with all improvements incorporated throughout
+
+INTEGRATION STRATEGY
+- Identify which suggestions target similar topics (e.g., setup, reproducibility, performance)
+- Group related improvements and apply them to the same document sections
+- For tutorial files: Enhance existing sections with all relevant suggestions, don't create duplicate sections
+- For documentation files: Merge suggestions into existing structure, avoid redundant sections
+- Result: ONE enhanced document that addresses all {total_suggestions} suggestions simultaneously
+
+CAPACITY AND SCOPE
+- You have enhanced token capacity to handle complex documents comprehensively
+- Tutorial documents: Enhanced capacity for step-by-step content, code examples, and comprehensive explanations
+- Complex documents: Increased capacity for multiple sections, detailed explanations, and extensive content
+- Comprehensive documents: Full capacity for complete documentation with all necessary sections
+
 STRICT CONSTRAINTS
-- Base the content solely on the evaluation report. Do not invent features, data, or claims not supported by it.
+- Base the content solely on the evaluation report and repo context. Do not invent features, data, or claims not supported by these sources.
+- CRITICAL: NEVER invent technical specifications including:
+  * Hardware requirements (RAM, CPU, disk space) unless explicitly stated in guidance/context
+  * Version numbers for dependencies unless explicitly stated in guidance/context
+  * Performance metrics, benchmarks, or timing estimates
+  * Biological/computational parameters or thresholds without evidence
+  * Installation commands or package names not found in the repo context
 - Prefer completeness and usability: produce the full file content, not just minimal "added" snippets.
 - Preserve top-of-file badges/logos if they exist in the original; keep title and header area intact unless the report requires changes.
 - CRITICAL: Preserve the original document structure, sections, and flow. Only enhance existing content and add missing information.
-- For tutorial files (.Rmd), maintain all original sections (Docker, installation methods, etc.) while improving clarity and adding missing details.
+- For tutorial files, maintain all original sections while improving clarity and adding missing details based on evaluation suggestions.
 - Fix obvious errors; improve structure and readability per report suggestions.
 - Include ONLY sections specifically requested by the evaluation report - do not add unnecessary sections.
 - Avoid redundancy: do not duplicate information across multiple sections.
 - ABSOLUTELY FORBIDDEN: Do NOT add summary sections, notes, conclusions, or any text at the end of documents
-- ABSOLUTELY FORBIDDEN: Do NOT wrap the entire document inside markdown code fences (```markdown). Do NOT start with ```markdown or end with ```. Return pure markdown content suitable for copy/paste.
+- ABSOLUTELY FORBIDDEN: Do NOT wrap the entire document inside markdown code fences (```markdown). Do NOT start with ```markdown or end with ```. Return pure content suitable for copy/paste.
 - ABSOLUTELY FORBIDDEN: Do NOT add phrases like "Happy analyzing!" or any concluding statements
 - Keep links well-formed; keep neutral, professional tone; concise, skimmable formatting.
-- For RMarkdown files (.Rmd), preserve YAML frontmatter exactly and do not wrap content in code fences.
+- Preserve file-specific formatting (e.g., YAML frontmatter, code fence syntax) and do not wrap content in extra code fences.
+
+COMPLETENESS REQUIREMENTS
+- Generate complete, comprehensive content that addresses all evaluation suggestions
+- For complex documents, ensure all sections are fully developed and detailed
+- For tutorial documents, include complete step-by-step instructions with examples
+- Use the increased token capacity to provide thorough, useful documentation
 
 OUTPUT
 - Return only the full markdown content for {target_file}. No commentary, no fences.
 """
 
 LLM_README_COMPREHENSIVE_PROMPT = """
-You are "BioGuider," a comprehensive documentation rewriter specializing in README files.
+You are "BioGuider," a comprehensive documentation rewriter specializing in README files with enhanced capacity for complex documentation.
 
 GOAL
-Create a complete, professional README.md that addresses all evaluation suggestions comprehensively. This is the main project documentation that users will see first.
+Create a complete, professional README.md that addresses all evaluation suggestions comprehensively. This is the main project documentation that users will see first. You now have increased token capacity to create thorough, comprehensive documentation.
 
 INPUTS (authoritative)
 - evaluation_report (structured JSON excerpts): <<{evaluation_report}>>
@@ -124,12 +158,26 @@ COMPREHENSIVE README REQUIREMENTS
 - Make it copy-paste ready for users
 - Use professional, clear language suitable for biomedical researchers
 
+ENHANCED CAPACITY FEATURES
+- You have increased token capacity to create comprehensive documentation
+- Include detailed explanations, multiple examples, and thorough coverage
+- Provide extensive installation instructions with platform-specific details
+- Add comprehensive usage examples with different scenarios
+- Include detailed API documentation if applicable
+- Provide troubleshooting guides with common issues and solutions
+
 STRICT CONSTRAINTS
 - Base the content solely on the evaluation report. Do not invent features, data, or claims not supported by it.
 - ABSOLUTELY FORBIDDEN: Do NOT wrap the entire document inside markdown code fences (```markdown). Return pure markdown content.
 - ABSOLUTELY FORBIDDEN: Do NOT add summary sections, notes, conclusions, or any text at the end of documents
 - Keep links well-formed; use neutral, professional tone; concise, skimmable formatting.
 
+COMPLETENESS REQUIREMENTS
+- Generate complete, comprehensive content that addresses all evaluation suggestions
+- Ensure all sections are fully developed and detailed
+- Use the increased token capacity to provide thorough, useful documentation
+- Include all necessary information for users to successfully use the software
+
 OUTPUT
 - Return only the full README.md content. No commentary, no fences.
 """
@@ -139,6 +187,322 @@ class LLMContentGenerator:
     def __init__(self, llm: BaseChatOpenAI):
         self.llm = llm
 
+    def _detect_truncation(self, content: str, target_file: str) -> bool:
+        """
+        Detect if content appears to be truncated based on common patterns.
+        Universal detection for all file types.
+        
+        Args:
+            content: Generated content to check
+            target_file: Target file path for context
+            
+        Returns:
+            True if content appears truncated, False otherwise
+        """
+        if not content or len(content.strip()) < 100:
+            return True
+        
+        # 1. Check for very short content (applies to all files)
+        # Only flag as truncated if content is very short (< 1500 chars)
+        if len(content) < 1500:
+            return True
+            
+        # 2. Check for incomplete code blocks (any language)
+        # Count opening and closing code fences
+        code_fence_count = content.count('```')
+        if code_fence_count > 0 and code_fence_count % 2 != 0:
+            # Unbalanced code fences suggest truncation
+            return True
+            
+        # 3. Check for specific language code blocks
+        if target_file.endswith('.Rmd'):
+            # R chunks should be complete
+            r_chunks_open = re.findall(r'```\{r[^}]*\}', content)
+            if r_chunks_open and not content.rstrip().endswith('```'):
+                # Has R chunks but doesn't end with closing fence
+                return True
+        
+        if target_file.endswith(('.py', '.js', '.ts', '.java', '.cpp', '.c')):
+            # Check for incomplete class/function definitions
+            lines = content.split('\n')
+            last_lines = [line.strip() for line in lines[-5:] if line.strip()]
+            if last_lines:
+                last_line = last_lines[-1]
+                if (last_line.endswith(':') or 
+                    last_line.endswith('{') or
+                    last_line.endswith('(') or
+                    'def ' in last_line or
+                    'class ' in last_line or
+                    'function ' in last_line):
+                    return True
+                    
+        # 4. Check for incomplete markdown sections (applies to all markdown-like files)
+        if any(target_file.endswith(ext) for ext in ['.md', '.Rmd', '.rst', '.txt']):
+            lines = content.split('\n')
+            last_non_empty_line = None
+            for line in reversed(lines):
+                if line.strip():
+                    last_non_empty_line = line.strip()
+                    break
+            
+            if last_non_empty_line:
+                # Check if last line looks incomplete
+                incomplete_endings = [
+                    '##',   # Header without content
+                    '###',  # Header without content  
+                    '####', # Header without content
+                    '-',    # List item
+                    '*',    # List item or emphasis
+                    ':',    # Definition or label
+                    '|',    # Table row
+                ]
+                
+                for ending in incomplete_endings:
+                    if last_non_empty_line.endswith(ending):
+                        return True
+                        
+                # Check if ends with incomplete patterns
+                content_end = content[-300:].strip().lower()
+                incomplete_patterns = [
+                    '## ',      # Section header without content
+                    '### ',     # Subsection without content
+                    '#### ',    # Sub-subsection without content
+                    '```{',     # Incomplete code chunk
+                    '```r',     # Incomplete R chunk
+                    '```python',# Incomplete Python chunk
+                ]
+                
+                for pattern in incomplete_patterns:
+                    if content_end.endswith(pattern.lower()):
+                        return True
+                    
+        return False
+
+    def _appears_complete(self, content: str, target_file: str) -> bool:
+        """
+        Check if content appears to be complete based on structure and patterns.
+        Universal completion check for all file types.
+        
+        Args:
+            content: Generated content to check
+            target_file: Target file path for context
+            
+        Returns:
+            True if content appears complete, False if it needs continuation
+        """
+        if not content or len(content.strip()) < 100:
+            return False
+        
+        # 1. Check for balanced code blocks (applies to all files)
+        code_block_count = content.count('```')
+        if code_block_count > 0 and code_block_count % 2 != 0:
+            # Unbalanced code blocks suggest incomplete
+            return False
+            
+        # 2. File type specific checks
+        
+        # RMarkdown files
+        if target_file.endswith('.Rmd'):
+            # Check for proper YAML frontmatter
+            if not content.startswith('---'):
+                return False
+                
+            # Check for conclusion patterns
+            conclusion_patterns = [
+                'sessionInfo()',
+                'session.info()',
+                '## Conclusion',
+                '## Summary',
+                '## Session Info',
+                '</details>',
+                'knitr::knit(',
+            ]
+            
+            content_lower = content.lower()
+            has_conclusion = any(pattern.lower() in content_lower for pattern in conclusion_patterns)
+            
+            # If we have a conclusion and balanced code blocks, likely complete
+            if has_conclusion and code_block_count > 0:
+                return True
+        
+        # Markdown files
+        if target_file.endswith('.md'):
+            # Check for conclusion sections
+            conclusion_patterns = [
+                '## Conclusion',
+                '## Summary',
+                '## Next Steps',
+                '## Further Reading',
+                '## References',
+                '## License',
+            ]
+            
+            content_lower = content.lower()
+            has_conclusion = any(pattern.lower() in content_lower for pattern in conclusion_patterns)
+            
+            if has_conclusion and len(content) > 2000:
+                return True
+        
+        # Python files
+        if target_file.endswith('.py'):
+            # Check for balanced brackets/parentheses
+            if content.count('(') != content.count(')'):
+                return False
+            if content.count('[') != content.count(']'):
+                return False
+            if content.count('{') != content.count('}'):
+                return False
+                
+            # Check for complete structure (reasonable length + proper ending)
+            lines = [line for line in content.split('\n') if line.strip()]
+            if len(lines) > 20:  # Has reasonable content
+                last_line = lines[-1].strip()
+                # Should not end with incomplete statements
+                if not (last_line.endswith(':') or 
+                       last_line.endswith('\\') or
+                       last_line.endswith(',')):
+                    return True
+        
+        # JavaScript/TypeScript files
+        if target_file.endswith(('.js', '.ts', '.jsx', '.tsx')):
+            # Check for balanced brackets
+            if content.count('{') != content.count('}'):
+                return False
+            if content.count('(') != content.count(')'):
+                return False
+                
+            lines = [line for line in content.split('\n') if line.strip()]
+            if len(lines) > 20:
+                last_line = lines[-1].strip()
+                # Complete if ends with proper syntax
+                if (last_line.endswith('}') or 
+                    last_line.endswith(';') or
+                    last_line.endswith('*/') or
+                    last_line.startswith('//')):
+                    return True
+        
+        # 3. Generic checks for all file types
+        if len(content) > 3000:  # Reasonable length
+            # Check if it ends with complete sentences/sections
+            lines = content.split('\n')
+            last_lines = [line.strip() for line in lines[-10:] if line.strip()]
+            
+            if last_lines:
+                last_line = last_lines[-1]
+                # Complete if ends with proper punctuation or closing tags
+                complete_endings = [
+                    '.',      # Sentence
+                    '```',    # Code block
+                    '---',    # Section divider
+                    '</details>',  # HTML details
+                    '}',      # Closing brace
+                    ';',      # Statement end
+                    '*/',     # Comment end
+                ]
+                
+                if any(last_line.endswith(ending) for ending in complete_endings):
+                    return True
+                    
+        return False
+
+    def _generate_continuation(self, target_file: str, evaluation_report: dict, 
+                             context: str, existing_content: str) -> tuple[str, dict]:
+        """
+        Generate continuation content from where previous generation left off.
+        
+        Args:
+            target_file: Target file path
+            evaluation_report: Evaluation report data
+            context: Repository context
+            existing_content: Previously generated content
+            
+        Returns:
+            Tuple of (continuation_content, token_usage)
+        """
+        # Create LLM for continuation (uses 16k tokens by default)
+        from bioguider.agents.agent_utils import get_llm
+        import os
+        
+        llm = get_llm(
+            api_key=os.environ.get("OPENAI_API_KEY"),
+            model_name=os.environ.get("OPENAI_MODEL", "gpt-4o"),
+            azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"),
+            api_version=os.environ.get("OPENAI_API_VERSION"),
+            azure_deployment=os.environ.get("OPENAI_DEPLOYMENT_NAME"),
+        )
+        
+        conv = CommonConversation(llm)
+        
+        # Calculate total suggestions for the prompt
+        total_suggestions = 1
+        if isinstance(evaluation_report, dict):
+            if "total_suggestions" in evaluation_report:
+                total_suggestions = evaluation_report["total_suggestions"]
+            elif "suggestions" in evaluation_report and isinstance(evaluation_report["suggestions"], list):
+                total_suggestions = len(evaluation_report["suggestions"])
+        
+        continuation_prompt = f"""
+You are "BioGuider," continuing a documentation generation task with enhanced capacity for complex documents.
+
+GOAL
+Continue generating the document "{target_file}" from where the previous generation left off. 
+The previous content was truncated and needs to be completed. You now have increased token 
+capacity to handle complex documents comprehensively.
+
+PREVIOUS CONTENT (do not repeat this):
+```
+{existing_content[-1000:]}  # Last 1000 chars for context
+```
+
+TASK
+Continue the document naturally from the last complete section. Maintain the same style, 
+structure, and flow as the previous content. Complete all remaining sections that should 
+be in this document.
+
+CAPACITY AND SCOPE
+- You have enhanced token capacity to handle complex documents comprehensively
+- Tutorial documents: Enhanced capacity for step-by-step content, code examples, and comprehensive explanations
+- Complex documents: Increased capacity for multiple sections, detailed explanations, and extensive content
+- Comprehensive documents: Full capacity for complete documentation with all necessary sections
+
+INPUTS
+- evaluation_report (contains {total_suggestions} suggestions to integrate): {json.dumps(evaluation_report)[:4000]}
+- context: {context[:2000]}
+
+REMINDER: SINGLE DOCUMENT APPROACH
+- The evaluation report contains {total_suggestions} SEPARATE suggestions
+- These should be integrated into ONE cohesive continuation
+- Do NOT create {total_suggestions} separate sections for each suggestion
+- Group related suggestions (e.g., setup, reproducibility, performance) and integrate them naturally
+
+REQUIREMENTS
+- Continue seamlessly from the previous content
+- Maintain the same tone and style
+- Complete all sections that should be in this document
+- Preserve file-specific formatting (e.g., YAML frontmatter, code block syntax appropriate to the language)
+- Do not repeat content already generated
+- Return only the continuation content, not the full document
+- Use the increased token capacity to provide thorough, complete content
+- NEVER invent technical specifications (hardware, versions, performance) unless explicitly in evaluation report or context
+- ABSOLUTELY FORBIDDEN: Do NOT wrap content in markdown code fences (```markdown). Return pure content only.
+- ABSOLUTELY FORBIDDEN: Do NOT add summary sections, notes, conclusions, or any text at the end of documents
+
+COMPLETENESS REQUIREMENTS
+- Generate complete, comprehensive content that addresses all remaining evaluation suggestions
+- For complex documents, ensure all sections are fully developed and detailed
+- For tutorial documents, include complete step-by-step instructions with examples
+- Use the increased token capacity to provide thorough, useful documentation
+
+OUTPUT
+Return only the continuation content that should be appended to the existing content.
+"""
+        
+        content, token_usage = conv.generate(
+            system_prompt=continuation_prompt, 
+            instruction_prompt="Continue the document from where it left off."
+        )
+        return content.strip(), token_usage
+
     def generate_section(self, suggestion: SuggestionItem, style: StyleProfile, context: str = "") -> tuple[str, dict]:
         conv = CommonConversation(self.llm)
         section_name = suggestion.anchor_hint or suggestion.category.split(".")[-1].replace("_", " ").title()
@@ -158,7 +522,43 @@ class LLMContentGenerator:
         return content.strip(), token_usage
 
     def generate_full_document(self, target_file: str, evaluation_report: dict, context: str = "") -> tuple[str, dict]:
-        conv = CommonConversation(self.llm)
+        # Create LLM (uses 16k tokens by default - enough for any document)
+        from bioguider.agents.agent_utils import get_llm
+        import os
+        import json
+        from datetime import datetime
+        
+        # Get LLM with default 16k token limit
+        llm = get_llm(
+            api_key=os.environ.get("OPENAI_API_KEY"),
+            model_name=os.environ.get("OPENAI_MODEL", "gpt-4o"),
+            azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"),
+            api_version=os.environ.get("OPENAI_API_VERSION"),
+            azure_deployment=os.environ.get("OPENAI_DEPLOYMENT_NAME"),
+        )
+        
+        conv = CommonConversation(llm)
+        
+        # Debug: Save generation settings and context
+        debug_info = {
+            "target_file": target_file,
+            "timestamp": datetime.now().isoformat(),
+            "evaluation_report": evaluation_report,
+            "context_length": len(context),
+            "llm_settings": {
+                "model_name": os.environ.get("OPENAI_MODEL", "gpt-4o"),
+                "azure_deployment": os.environ.get("OPENAI_DEPLOYMENT_NAME"),
+                "max_tokens": getattr(llm, 'max_tokens', 16384)
+            }
+        }
+        
+        # Save debug info to file
+        debug_dir = "outputs/debug_generation"
+        os.makedirs(debug_dir, exist_ok=True)
+        safe_filename = target_file.replace("/", "_").replace(".", "_")
+        debug_file = os.path.join(debug_dir, f"{safe_filename}_debug.json")
+        with open(debug_file, 'w', encoding='utf-8') as f:
+            json.dump(debug_info, f, indent=2, ensure_ascii=False)
         
         # Use comprehensive README prompt for README.md files
         if target_file.endswith("README.md"):
@@ -168,13 +568,204 @@ class LLMContentGenerator:
                 context=context[:4000],
             )
         else:
+            # Calculate total suggestions for the prompt
+            total_suggestions = 1
+            if isinstance(evaluation_report, dict):
+                if "total_suggestions" in evaluation_report:
+                    total_suggestions = evaluation_report["total_suggestions"]
+                elif "suggestions" in evaluation_report and isinstance(evaluation_report["suggestions"], list):
+                    total_suggestions = len(evaluation_report["suggestions"])
+            
             system_prompt = LLM_FULLDOC_PROMPT.format(
                 target_file=target_file,
                 evaluation_report=json.dumps(evaluation_report)[:6000],
                 context=context[:4000],
+                total_suggestions=total_suggestions,
             )
         
+        # Save initial prompt for debugging
+        prompt_file = os.path.join(debug_dir, f"{safe_filename}_prompt.txt")
+        with open(prompt_file, 'w', encoding='utf-8') as f:
+            f.write("=== SYSTEM PROMPT ===\n")
+            f.write(system_prompt)
+            f.write("\n\n=== INSTRUCTION PROMPT ===\n")
+            f.write("Write the full document now.")
+            f.write("\n\n=== EVALUATION REPORT ===\n")
+            f.write(json.dumps(evaluation_report, indent=2))
+            f.write("\n\n=== CONTEXT ===\n")
+            f.write(context[:2000] + "..." if len(context) > 2000 else context)
+        
+        # Initial generation
         content, token_usage = conv.generate(system_prompt=system_prompt, instruction_prompt="Write the full document now.")
-        return content.strip(), token_usage
+        content = content.strip()
+        
+        # Save initial generation for debugging
+        generation_file = os.path.join(debug_dir, f"{safe_filename}_generation_0.txt")
+        with open(generation_file, 'w', encoding='utf-8') as f:
+            f.write(f"=== INITIAL GENERATION ===\n")
+            f.write(f"Tokens: {token_usage}\n")
+            f.write(f"Length: {len(content)} characters\n")
+            f.write(f"Truncation detected: {self._detect_truncation(content, target_file)}\n")
+            f.write(f"\n=== CONTENT ===\n")
+            f.write(content)
+        
+        # Check for truncation and continue if needed
+        max_continuations = 3  # Limit to prevent infinite loops
+        continuation_count = 0
+        
+        while (self._detect_truncation(content, target_file) and 
+               continuation_count < max_continuations):
+            
+            # Additional check: if content appears complete, don't continue
+            if self._appears_complete(content, target_file):
+                break
+            continuation_count += 1
+            
+            # Save continuation prompt for debugging
+            continuation_prompt_file = os.path.join(debug_dir, f"{safe_filename}_continuation_{continuation_count}_prompt.txt")
+            continuation_prompt = f"""
+You are "BioGuider," continuing a documentation generation task with enhanced capacity for complex documents.
+
+GOAL
+Continue generating the document "{target_file}" from where the previous generation left off. 
+The previous content was truncated and needs to be completed. You now have increased token 
+capacity to handle complex documents comprehensively.
+
+PREVIOUS CONTENT (do not repeat this):
+```
+{content[-1000:]}  # Last 1000 chars for context
+```
+
+TASK
+Continue the document naturally from the last complete section. Maintain the same style, 
+structure, and flow as the previous content. Complete all remaining sections that should 
+be in this document.
+
+CRITICAL REQUIREMENTS:
+- Do NOT repeat any content already generated above
+- Do NOT duplicate sections, headers, or code blocks that already exist
+- Generate ONLY new, unique content that continues from where the previous content ended
+- If the previous content appears complete, add complementary sections that enhance the document
+- Focus on adding missing sections, examples, or explanations that weren't covered
+
+CAPACITY AND SCOPE
+- You have enhanced token capacity to handle complex documents comprehensively
+- Tutorial documents: Enhanced capacity for step-by-step content, code examples, and comprehensive explanations
+- Complex documents: Increased capacity for multiple sections, detailed explanations, and extensive content
+- Comprehensive documents: Full capacity for complete documentation with all necessary sections
+
+INPUTS
+- evaluation_report (contains {total_suggestions} suggestions to integrate): {json.dumps(evaluation_report)[:4000]}
+- context: {context[:2000]}
+
+REMINDER: SINGLE DOCUMENT APPROACH
+- The evaluation report contains {total_suggestions} SEPARATE suggestions
+- These should be integrated into ONE cohesive continuation
+- Do NOT create {total_suggestions} separate sections for each suggestion
+- Group related suggestions (e.g., setup, reproducibility, performance) and integrate them naturally
+
+REQUIREMENTS
+- Continue seamlessly from the previous content
+- Maintain the same tone and style
+- Complete all sections that should be in this document
+- Preserve file-specific formatting (e.g., YAML frontmatter, code block syntax appropriate to the language)
+- Do not repeat content already generated
+- Return only the continuation content, not the full document
+- Use the increased token capacity to provide thorough, complete content
+- NEVER invent technical specifications (hardware, versions, performance) unless explicitly in evaluation report or context
+- ABSOLUTELY FORBIDDEN: Do NOT wrap content in markdown code fences (```markdown). Return pure content only.
+- ABSOLUTELY FORBIDDEN: Do NOT add summary sections, notes, conclusions, or any text at the end of documents
+
+COMPLETENESS REQUIREMENTS
+- Generate complete, comprehensive content that addresses all remaining evaluation suggestions
+- For complex documents, ensure all sections are fully developed and detailed
+- For tutorial documents, include complete step-by-step instructions with examples
+- Use the increased token capacity to provide thorough, useful documentation
+
+OUTPUT
+- Return only the continuation content. No commentary, no fences.
+"""
+            
+            with open(continuation_prompt_file, 'w', encoding='utf-8') as f:
+                f.write(continuation_prompt)
+            
+            # Generate continuation
+            continuation_content, continuation_usage = self._generate_continuation(
+                target_file=target_file,
+                evaluation_report=evaluation_report,
+                context=context,
+                existing_content=content
+            )
+            
+            # Save continuation generation for debugging
+            continuation_file = os.path.join(debug_dir, f"{safe_filename}_continuation_{continuation_count}.txt")
+            with open(continuation_file, 'w', encoding='utf-8') as f:
+                f.write(f"=== CONTINUATION {continuation_count} ===\n")
+                f.write(f"Tokens: {continuation_usage}\n")
+                f.write(f"Length: {len(continuation_content)} characters\n")
+                f.write(f"Truncation detected: {self._detect_truncation(continuation_content, target_file)}\n")
+                f.write(f"\n=== CONTENT ===\n")
+                f.write(continuation_content)
+            
+            # Merge continuation with existing content
+            if continuation_content:
+                content += "\n\n" + continuation_content
+                # Update token usage
+                token_usage = {
+                    "total_tokens": token_usage.get("total_tokens", 0) + continuation_usage.get("total_tokens", 0),
+                    "prompt_tokens": token_usage.get("prompt_tokens", 0) + continuation_usage.get("prompt_tokens", 0),
+                    "completion_tokens": token_usage.get("completion_tokens", 0) + continuation_usage.get("completion_tokens", 0),
+                }
+                
+                # Save merged content for debugging
+                merged_file = os.path.join(debug_dir, f"{safe_filename}_merged_{continuation_count}.txt")
+                with open(merged_file, 'w', encoding='utf-8') as f:
+                    f.write(f"=== MERGED CONTENT AFTER CONTINUATION {continuation_count} ===\n")
+                    f.write(f"Total length: {len(content)} characters\n")
+                    f.write(f"Truncation detected: {self._detect_truncation(content, target_file)}\n")
+                    f.write(f"\n=== CONTENT ===\n")
+                    f.write(content)
+            else:
+                # If continuation is empty, break to avoid infinite loop
+                break
+        
+        # Clean up any markdown code fences that might have been added
+        content = self._clean_markdown_fences(content)
+        
+        # Save final cleaned content for debugging
+        final_file = os.path.join(debug_dir, f"{safe_filename}_final.txt")
+        with open(final_file, 'w', encoding='utf-8') as f:
+            f.write(f"=== FINAL CLEANED CONTENT ===\n")
+            f.write(f"Total tokens: {token_usage}\n")
+            f.write(f"Final length: {len(content)} characters\n")
+            f.write(f"Continuations used: {continuation_count}\n")
+            f.write(f"\n=== CONTENT ===\n")
+            f.write(content)
+        
+        return content, token_usage
+    
+    def _clean_markdown_fences(self, content: str) -> str:
+        """
+        Remove markdown code fences that shouldn't be in the final content.
+        """
+        # Remove ```markdown at the beginning
+        if content.startswith('```markdown\n'):
+            content = content[12:]  # Remove ```markdown\n
+        
+        # Remove ``` at the end
+        if content.endswith('\n```'):
+            content = content[:-4]  # Remove \n```
+        elif content.endswith('```'):
+            content = content[:-3]  # Remove ```
+        
+        # Remove any standalone ```markdown lines
+        lines = content.split('\n')
+        cleaned_lines = []
+        for line in lines:
+            if line.strip() == '```markdown':
+                continue
+            cleaned_lines.append(line)
+        
+        return '\n'.join(cleaned_lines)
 
 

bioguider/managers/evaluation_manager.py

@@ -3,10 +3,8 @@ from pathlib import Path
 
 from bioguider.agents.evaluation_tutorial_task import EvaluationTutorialTask
 from bioguider.agents.evaluation_userguide_task import EvaluationUserGuideTask
-from bioguider.agents.prompt_utils import CollectionGoalItemEnum
 from bioguider.database.code_structure_db import CodeStructureDb
 from bioguider.utils.constants import ProjectMetadata
-from bioguider.utils.gitignore_checker import GitignoreChecker
 
 from ..agents.identification_task import IdentificationTask
 from ..rag.rag import RAG

bioguider/managers/generation_manager.py

@@ -34,6 +34,7 @@ class DocumentationGenerationManager:
         self.output = OutputManager(base_outputs_dir=output_dir)
         self.llm_gen = LLMContentGenerator(llm)
         self.llm_cleaner = LLMCleaner(llm)
+        
 
     def print_step(self, step_name: str | None = None, step_output: str | None = None):
         if self.step_callback is None:
@@ -143,54 +144,175 @@ class DocumentationGenerationManager:
             self.print_step(step_name="ProcessingFile", step_output=f"Processing {fpath} ({processed_files}/{total_files}) - {len(edits)} edits")
             
             original_content = files.get(fpath, "")
+            
+            # Group suggestions by file to avoid duplicate generation
+            file_suggestions = []
+            full_replace_edits = []
+            section_edits = []
+            
+            for e in edits:
+                suggestion = next((s for s in suggestions if s.id == e.suggestion_id), None) if e.suggestion_id else None
+                if suggestion:
+                    file_suggestions.append(suggestion)
+                    if e.edit_type == "full_replace":
+                        full_replace_edits.append(e)
+                    else:
+                        section_edits.append(e)
+            
+            # Debug: Save suggestion grouping info
+            import json
+            import os
+            from datetime import datetime
+            
+            debug_dir = "outputs/debug_generation"
+            os.makedirs(debug_dir, exist_ok=True)
+            safe_filename = fpath.replace("/", "_").replace(".", "_")
+            
+            grouping_info = {
+                "file_path": fpath,
+                "total_edits": len(edits),
+                "file_suggestions_count": len(file_suggestions),
+                "full_replace_edits_count": len(full_replace_edits),
+                "section_edits_count": len(section_edits),
+                "suggestions": [
+                    {
+                        "id": s.id,
+                        "category": s.category,
+                        "content_guidance": s.content_guidance[:200] + "..." if len(s.content_guidance or "") > 200 else s.content_guidance,
+                        "target_files": s.target_files
+                    } for s in file_suggestions
+                ],
+                "timestamp": datetime.now().isoformat()
+            }
+            
+            grouping_file = os.path.join(debug_dir, f"{safe_filename}_grouping.json")
+            with open(grouping_file, 'w', encoding='utf-8') as f:
+                json.dump(grouping_info, f, indent=2, ensure_ascii=False)
+            
             content = original_content
             total_stats = {"added_lines": 0}
-            for e in edits:
-                context = original_content
-                if not e.content_template or e.content_template.strip() == "":
-                    # Generate LLM content - use full document generation for full_replace, section generation for others
-                    suggestion = next((s for s in suggestions if s.id == e.suggestion_id), None) if e.suggestion_id else None
-                    if suggestion:
-                        if e.edit_type == "full_replace":
-                            self.print_step(step_name="GeneratingContent", step_output=f"Generating full document for {e.suggestion_id} using LLM...")
+            
+            # CRITICAL: Generate content ONCE per file if there are full_replace edits
+            # All suggestions for this file are merged into a single evaluation report
+            # This prevents duplicate content generation
+            if full_replace_edits:
+                self.print_step(
+                    step_name="GeneratingContent", 
+                    step_output=f"🔄 Generating full document for {fpath} with {len(file_suggestions)} suggestions using LLM (SINGLE CALL)..."
+                )
+                
+                # Merge all suggestions for this file into a single evaluation report
+                # Format suggestions with clear numbering to help LLM understand they're separate improvements
+                suggestions_list = []
+                for idx, s in enumerate(file_suggestions, 1):
+                    suggestions_list.append({
+                        "suggestion_number": idx,
+                        "category": s.category if hasattr(s, 'category') else "general",
+                        "content_guidance": s.content_guidance,
+                        "evidence": s.source.get("evidence", "") if s.source else ""
+                    })
+                
+                merged_evaluation_report = {
+                    "total_suggestions": len(file_suggestions),
+                    "integration_instruction": f"Integrate ALL {len(file_suggestions)} suggestions below into ONE cohesive document. Do NOT create {len(file_suggestions)} separate versions.",
+                    "suggestions": suggestions_list
+                }
+                
+                # Debug: Save merged evaluation report
+                merged_report_file = os.path.join(debug_dir, f"{safe_filename}_merged_report.json")
+                with open(merged_report_file, 'w', encoding='utf-8') as f:
+                    json.dump(merged_evaluation_report, f, indent=2, ensure_ascii=False)
+                
+                # Debug: Log that we're about to make a single generation call
+                debug_log_file = os.path.join(debug_dir, f"{safe_filename}_generation_log.txt")
+                with open(debug_log_file, 'a', encoding='utf-8') as f:
+                    f.write(f"\n=== GENERATION CALL at {datetime.now().isoformat()} ===\n")
+                    f.write(f"File: {fpath}\n")
+                    f.write(f"Full replace edits: {len(full_replace_edits)}\n")
+                    f.write(f"Total suggestions: {len(file_suggestions)}\n")
+                    f.write(f"Merged into single call: YES\n")
+                    f.write(f"Suggestion IDs: {[s.id for s in file_suggestions]}\n\n")
+                
+                gen_content, gen_usage = self.llm_gen.generate_full_document(
+                    target_file=fpath,
+                    evaluation_report=merged_evaluation_report,
+                    context=original_content,
+                )
+                
+                # Debug: Log completion
+                with open(debug_log_file, 'a', encoding='utf-8') as f:
+                    f.write(f"Generation completed at {datetime.now().isoformat()}\n")
+                    f.write(f"Content length: {len(gen_content) if isinstance(gen_content, str) else 0} characters\n")
+                    f.write(f"Tokens used: {gen_usage.get('total_tokens', 0)}\n")
+                    f.write(f"SUCCESS: {isinstance(gen_content, str) and gen_content}\n\n")
+                
+                if isinstance(gen_content, str) and gen_content:
+                    self.print_step(step_name="LLMFullDoc", step_output=f"✓ Generated full document for {fpath} ({gen_usage.get('total_tokens', 0)} tokens)")
+                    # Apply the generated content to all full_replace edits
+                    for e in full_replace_edits:
+                        e.content_template = gen_content
+                    content = gen_content
+                else:
+                    # Fallback: try individual generation but only for the first edit to avoid duplicates
+                    if full_replace_edits:
+                        e = full_replace_edits[0]  # Only process the first edit
+                        suggestion = next((s for s in suggestions if s.id == e.suggestion_id), None) if e.suggestion_id else None
+                        if suggestion and (not e.content_template or e.content_template.strip() == ""):
+                            self.print_step(step_name="GeneratingContent", step_output=f"Fallback: Generating full document for {e.suggestion_id} using LLM...")
                             gen_content, gen_usage = self.llm_gen.generate_full_document(
                                 target_file=e.file_path,
                                 evaluation_report={"suggestion": suggestion.content_guidance, "evidence": suggestion.source.get("evidence", "") if suggestion.source else ""},
-                                context=context,
+                                context=original_content,
                             )
                             if isinstance(gen_content, str) and gen_content:
                                 self.print_step(step_name="LLMFullDoc", step_output=f"✓ Generated full document for {e.suggestion_id} ({gen_usage.get('total_tokens', 0)} tokens)")
-                                e.content_template = gen_content
-                        else:
-                            self.print_step(step_name="GeneratingContent", step_output=f"Generating section for {e.suggestion_id} using LLM...")
-                            gen_section, gen_usage = self.llm_gen.generate_section(
-                                suggestion=suggestion,
-                                style=plan.style_profile,
-                                context=context,
-                            )
-                            if isinstance(gen_section, str) and gen_section:
-                                self.print_step(step_name="LLMSection", step_output=f"✓ Generated section for {e.suggestion_id} ({gen_usage.get('total_tokens', 0)} tokens)")
-                                # Ensure header present
-                                if gen_section.lstrip().startswith("#"):
-                                    e.content_template = gen_section
-                                else:
-                                    title = e.anchor.get('value', '').strip() or ''
-                                    e.content_template = f"## {title}\n\n{gen_section}" if title else gen_section
-                content, stats = self.renderer.apply_edit(content, e)
-                # After applying full document or section changes, run a general cleaner pass for all text files
-                # to fix markdown/formatting issues without changing meaning.
-                try:
-                    if fpath.endswith((".md", ".rst", ".Rmd", ".Rd")) and content:
-                        self.print_step(step_name="CleaningContent", step_output=f"Cleaning formatting for {fpath}...")
-                        cleaned, _usage = self.llm_cleaner.clean_readme(content)
-                        if isinstance(cleaned, str) and cleaned.strip():
-                            content = cleaned
+                                # Apply the same content to all full_replace edits
+                                for edit in full_replace_edits:
+                                    edit.content_template = gen_content
+                                content = gen_content
+            else:
+                # Handle section edits individually
+                for e in section_edits:
+                    suggestion = next((s for s in suggestions if s.id == e.suggestion_id), None) if e.suggestion_id else None
+                    if suggestion and (not e.content_template or e.content_template.strip() == ""):
+                        self.print_step(step_name="GeneratingContent", step_output=f"Generating section for {e.suggestion_id} using LLM...")
+                        gen_section, gen_usage = self.llm_gen.generate_section(
+                            suggestion=suggestion,
+                            style=plan.style_profile,
+                            context=original_content,
+                        )
+                        if isinstance(gen_section, str) and gen_section:
+                            self.print_step(step_name="LLMSection", step_output=f"✓ Generated section for {e.suggestion_id} ({gen_usage.get('total_tokens', 0)} tokens)")
+                            # Ensure header present
+                            if gen_section.lstrip().startswith("#"):
+                                e.content_template = gen_section
+                            else:
+                                title = e.anchor.get('value', '').strip() or ''
+                                e.content_template = f"## {title}\n\n{gen_section}" if title else gen_section
+                    
+                    content, stats = self.renderer.apply_edit(content, e)
+                    total_stats["added_lines"] = total_stats.get("added_lines", 0) + stats.get("added_lines", 0)
+            
+            # Apply remaining edits that weren't full_replace
+            for e in edits:
+                if e.edit_type != "full_replace":
+                    content, stats = self.renderer.apply_edit(content, e)
+                    total_stats["added_lines"] = total_stats.get("added_lines", 0) + stats.get("added_lines", 0)
+            
+            # After applying full document or section changes, run a general cleaner pass for all text files
+            # to fix markdown/formatting issues without changing meaning.
+            try:
+                if fpath.endswith((".md", ".rst", ".Rmd", ".Rd")) and content:
+                    self.print_step(step_name="CleaningContent", step_output=f"Cleaning formatting for {fpath}...")
+                    cleaned, _usage = self.llm_cleaner.clean_readme(content)
+                    if isinstance(cleaned, str) and cleaned.strip():
+                        content = cleaned
+                    
+                    # LLM cleaner now handles markdown fences and unwanted summaries
                         
-                        # LLM cleaner now handles markdown fences and unwanted summaries
-                            
-                except Exception:
-                    pass
-                total_stats["added_lines"] = total_stats.get("added_lines", 0) + stats.get("added_lines", 0)
+            except Exception:
+                pass
+            
             revised[fpath] = content
             diff_stats[fpath] = total_stats
             self.print_step(step_name="RenderedFile", step_output=f"✓ Completed {fpath} - added {total_stats['added_lines']} lines")

bioguider/utils/code_structure_builder.py

@@ -6,6 +6,7 @@ from bioguider.utils.r_file_handler import RFileHandler
 from .gitignore_checker import GitignoreChecker
 from .python_file_handler import PythonFileHandler
 from ..database.code_structure_db import CodeStructureDb
+from ..rag.config import configs
 
 logger = logging.getLogger(__name__)
 
@@ -17,7 +18,12 @@ class CodeStructureBuilder:
         code_structure_db: CodeStructureDb,
     ):
         self.repo_path = str(repo_path)
-        self.gitignore_checker = GitignoreChecker(repo_path, str(gitignore_path))
+        self.gitignore_checker = GitignoreChecker(
+            directory=repo_path,
+            gitignore_path=str(gitignore_path),
+            exclude_dir_patterns=configs["file_filters"]["excluded_dirs"],
+            exclude_file_patterns=configs["file_filters"]["excluded_files"],
+        )
         self.file_handler = PythonFileHandler(repo_path)
         self.code_structure_db = code_structure_db
 

bioguider/utils/r_file_handler.py

@@ -348,10 +348,12 @@ class RFileHandler:
             s = line.lstrip()
             if s.startswith("#'"):
                 buf.append(s[2:].lstrip())
-                line_idx -= 1
-                continue
-            # stop at first non-roxygen line (don’t cross blank + NULL padding blocks)
-            break
+            elif s.strip() == "":
+                pass
+            else:
+                # stop at first non-roxygen line (don’t cross blank + NULL padding blocks)
+                break
+            line_idx -= 1
         if not buf:
             return None
         buf.reverse()

{bioguider-0.2.31.dist-info → bioguider-0.2.33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: bioguider
-Version: 0.2.31
+Version: 0.2.33
 Summary: An AI-Powered package to help biomedical developers to generate clear documentation
 License: MIT
 Author: Cankun Wang

{bioguider-0.2.31.dist-info → bioguider-0.2.33.dist-info}/RECORD

@@ -2,7 +2,7 @@ bioguider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bioguider/agents/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bioguider/agents/agent_task.py,sha256=TL0Zx8zOmiAVslmNbfMPQ38qTQ73QospY6Dwrwf8POg,2890
 bioguider/agents/agent_tools.py,sha256=r21wHV6a-Ic2T0dk4YzA-_d7PodHPM3GzRxJqv-llSw,7286
-bioguider/agents/agent_utils.py,sha256=Mj6yr_2y4veWokXrXlAsaP38Ez9sdnZruM8ZnnpjxQ4,14825
+bioguider/agents/agent_utils.py,sha256=VL6ui13v0Fo6FGRvrak6Q04q1q7LWuKY6o0JP4fx8Os,16293
 bioguider/agents/collection_execute_step.py,sha256=jE_oSQZI5WDaz0bJjUWoAfqWfVbGUqN--cvITSWCGiI,5614
 bioguider/agents/collection_observe_step.py,sha256=1xOw6N3uIoyh4h4_vcULAc5x5KZ9G-zZo42AhRidyn8,5373
 bioguider/agents/collection_plan_step.py,sha256=Nn0f8AOkEDCDtnhaqE7yCQoi7PVpsHmiUcsIqC0T0dQ,5956
@@ -23,7 +23,7 @@ bioguider/agents/dockergeneration_plan_step.py,sha256=SB8tQM9PkIKsD2o1DFD7bedcxz
 bioguider/agents/dockergeneration_task.py,sha256=mYmorLKnJ-Jku3Qq_Y_kcSTsbYIo3RiVdD0puxqXY5Q,6221
 bioguider/agents/dockergeneration_task_utils.py,sha256=v7emqrJlVW-A5ZdLmPSdiaMSKCR8uzy9UYzx_1cgzyo,9041
 bioguider/agents/evaluation_installation_task.py,sha256=0RNH5NV7YKwn_we_d3IjnFf_ee9IPCEQ_olebq2y8Ng,12130
-bioguider/agents/evaluation_readme_task.py,sha256=pi3oAGJgZhJgJG1xLgiobrk3Uy2a_JIarD5QSPBkmHA,30647
+bioguider/agents/evaluation_readme_task.py,sha256=GUrGZUK9Jvb82M0AooLRi7CeOpZ19t-YQfs-UxYKK2Q,30841
 bioguider/agents/evaluation_submission_requirements_task.py,sha256=J_6C-M2AfYue2C-gWBHl7KqGrTBuFBn9zmMV5vSRk-U,7834
 bioguider/agents/evaluation_task.py,sha256=uu0BjalctF9hQjGtT53whbeJHv2RVvs8_2woVUmOLRE,8132
 bioguider/agents/evaluation_tutorial_task.py,sha256=cxzXf5cXdZtYd8yCpffTlDGuZFx5NZ_iz9kkNXLLtns,10138
@@ -46,7 +46,7 @@ bioguider/generation/__init__.py,sha256=esV02QgCsY67-HBwSHDbA5AcbKzNRIT3wDwwh6N4
 bioguider/generation/change_planner.py,sha256=0N10jvkfn2J9b598FKOKPQecwmQv68yeuUvMZn81nOI,9715
 bioguider/generation/document_renderer.py,sha256=Md8NMo0CXNIqatWOdKE-_4k02Y3T_BCLmEPLTEiYUCA,7984
 bioguider/generation/llm_cleaner.py,sha256=qFgS5xi7bBO8HAJ9WFNzH3p9AhOsAkYjchKQHuAUWWM,2917
-bioguider/generation/llm_content_generator.py,sha256=UbRURH6RdEWBVVqQi96SlTkNEOt01yyuxr76jR8__GA,10983
+bioguider/generation/llm_content_generator.py,sha256=DEgk4uAgZrxBTVEN3ZuhL7W-tBfXOyn2X4e9rM1Gfhc,39748
 bioguider/generation/llm_injector.py,sha256=bVxP6Asv2em4MBOB5yFsS14AuaeT7NLKQQMcsEqXjPY,17352
 bioguider/generation/models.py,sha256=MlJOLjPHk8xs-UGW-TGN_M9cevTuxTG4tjm1d1L15go,2699
 bioguider/generation/output_manager.py,sha256=uwLyavND4kXOHlsXB0Berab3y8u6bhaEmQOQLl7wDAM,1963
@@ -55,8 +55,8 @@ bioguider/generation/report_loader.py,sha256=bxajeTDxod36iFsbSZhXSQjotxqP7LuAg5M
 bioguider/generation/style_analyzer.py,sha256=Vn9FAK1qJBNLolLC1tz362k4UBaPl107BlvkQc8pV2I,983
 bioguider/generation/suggestion_extractor.py,sha256=kkPOYE6FXRtYlogV0GQdBraZZJm08I6Oux5YKGUF1UU,29442
 bioguider/generation/test_metrics.py,sha256=ACXmSZc2L_UkkmC5h2s4tG44MXW1d-hClFwPCD5_BFI,7505
-bioguider/managers/evaluation_manager.py,sha256=EoZ8V4rmx16zk1J3N9cNjeo0aCa7i32fLEQ3b2UolEQ,5917
-bioguider/managers/generation_manager.py,sha256=GqjCci2eWHnIcJ-SOn5-hEMi8p3Jk4Q5E4KSObKELPs,31945
+bioguider/managers/evaluation_manager.py,sha256=7WlshfnqFkk34dDlf50qAINK5sFTaoCFE0f0vGYyRdc,5789
+bioguider/managers/generation_manager.py,sha256=m6hGu9_1HcL1YS0PMoFfXfVgDqsps1ahcj6L9E4jtoo,38636
 bioguider/managers/generation_test_manager.py,sha256=3mOBzQVpsLo_LpSspJcofn3CNtvgagS1DMr9Zuwkzq4,5307
 bioguider/rag/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bioguider/rag/config.py,sha256=5g4IqTzgyfZfax9Af9CTkXShgItPOt4_9TEMSekCPik,4602
@@ -64,7 +64,7 @@ bioguider/rag/data_pipeline.py,sha256=hyBmjOpP1ka_y_4X0lUwlNKEBjmPNOmETEfQf-s86Z
 bioguider/rag/embedder.py,sha256=jofR8hOj3Aj2IyBQ9y6FeAc84tgq5agbIfCGyFxYpJ8,650
 bioguider/rag/rag.py,sha256=JFPwrJlKDSyd3U3Gce_NSxI5343eNUbqPG9Fs5Pfoq0,4696
 bioguider/settings.py,sha256=BD_iz9aYarxmWUl0XaKl4-D4oTXMhFzljsXLNn2phis,3143
-bioguider/utils/code_structure_builder.py,sha256=PVeooVKxkiBEkX_VEu8h1ACGs-ynrgk_ce2FKJfeeZc,1947
+bioguider/utils/code_structure_builder.py,sha256=t7Ao3r50v7c0JrS74J70mCOtaEFnKvam-bsA7OSvFQo,2192
 bioguider/utils/constants.py,sha256=NGmqEgxNDL1fe-htJbtHGcU94EVUK28YAupxGYOJO_c,9012
 bioguider/utils/default.gitignore,sha256=XjPdyO2KV8z8iyuqluaNR_70tBQftMpyKL8HboVNyeI,1605
 bioguider/utils/file_utils.py,sha256=DOWRluneNpGQ4uVwwX9Tp2VzmZ56wIqXKMyjcMH9Bpc,6229
@@ -72,9 +72,9 @@ bioguider/utils/gitignore_checker.py,sha256=pOYUwsS9D5014LxcZb0cj3s2CAYaD2uF_pYJ
 bioguider/utils/notebook_utils.py,sha256=SfU1iLuwgbDzNN-TUh_qbnfUSgn-PI6NrK6QfmdpMqQ,4009
 bioguider/utils/pyphen_utils.py,sha256=cdZc3qphkvMDeL5NiZ8Xou13M_uVNP7ifJ-FwxO-0BE,2680
 bioguider/utils/python_file_handler.py,sha256=BERiE2RHxpu3gAzv26jr8ZQetkrtnMZOv9SjpQ7WIdg,2650
-bioguider/utils/r_file_handler.py,sha256=8HpFaYKP8N1nItwr9tOx49m99pcLSt8EUtTNTJ7xNoE,19564
+bioguider/utils/r_file_handler.py,sha256=y57Y04wjgtFWve0lPg1EOrNNOccPfnNF0z2WnlFMX74,19616
 bioguider/utils/utils.py,sha256=h8OhCjzLpHkb3ndnjRBUOBHD7csbHdEVNXf75SRN8Zc,4413
-bioguider-0.2.31.dist-info/LICENSE,sha256=qzkvZcKwwA5DuSuhXMOm2LcO6BdEr4V7jwFZVL2-jL4,1065
-bioguider-0.2.31.dist-info/METADATA,sha256=k5RSl8D1JHB5mFSCWqcJjTLF1WDDhtBw2BbFfmg-Trw,1962
-bioguider-0.2.31.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-bioguider-0.2.31.dist-info/RECORD,,
+bioguider-0.2.33.dist-info/LICENSE,sha256=qzkvZcKwwA5DuSuhXMOm2LcO6BdEr4V7jwFZVL2-jL4,1065
+bioguider-0.2.33.dist-info/METADATA,sha256=Yyqyvrm_CLNHy1fgDluqbCsePUqA_mYwkzRupFTHbRU,1962
+bioguider-0.2.33.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+bioguider-0.2.33.dist-info/RECORD,,