deepcode-hku 1.0.1__py3-none-any.whl

utils/file_processor.py

@@ -0,0 +1,426 @@
+"""
+File processing utilities for handling paper files and related operations.
+"""
+
+import json
+import os
+import re
+from typing import Dict, List, Optional, Union
+
+
+class FileProcessor:
+    """
+    A class to handle file processing operations including path extraction and file reading.
+    """
+
+    @staticmethod
+    def extract_file_path(file_info: Union[str, Dict]) -> Optional[str]:
+        """
+        Extract paper directory path from the input information.
+
+        Args:
+            file_info: Either a JSON string or a dictionary containing file information
+
+        Returns:
+            Optional[str]: The extracted paper directory path or None if not found
+        """
+        try:
+            # Handle direct file path input
+            if isinstance(file_info, str):
+                # Check if it's a file path (existing or not)
+                if file_info.endswith(
+                    (".md", ".pdf", ".txt", ".docx", ".doc", ".html", ".htm")
+                ):
+                    # It's a file path, return the directory
+                    return os.path.dirname(os.path.abspath(file_info))
+                elif os.path.exists(file_info):
+                    if os.path.isfile(file_info):
+                        return os.path.dirname(os.path.abspath(file_info))
+                    elif os.path.isdir(file_info):
+                        return os.path.abspath(file_info)
+
+                # Try to parse as JSON
+                try:
+                    info_dict = json.loads(file_info)
+                except json.JSONDecodeError:
+                    # 尝试从文本中提取JSON
+                    info_dict = FileProcessor.extract_json_from_text(file_info)
+                    if not info_dict:
+                        # If not JSON and doesn't look like a file path, raise error
+                        raise ValueError(
+                            f"Input is neither a valid file path nor JSON: {file_info}"
+                        )
+            else:
+                info_dict = file_info
+
+            # Extract paper path from dictionary
+            paper_path = info_dict.get("paper_path")
+            if not paper_path:
+                raise ValueError("No paper_path found in input dictionary")
+
+            # Get the directory path instead of the file path
+            paper_dir = os.path.dirname(paper_path)
+
+            # Convert to absolute path if relative
+            if not os.path.isabs(paper_dir):
+                paper_dir = os.path.abspath(paper_dir)
+
+            return paper_dir
+
+        except (AttributeError, TypeError) as e:
+            raise ValueError(f"Invalid input format: {str(e)}")
+
+    @staticmethod
+    def find_markdown_file(directory: str) -> Optional[str]:
+        """
+        Find the first markdown file in the given directory.
+
+        Args:
+            directory: Directory path to search
+
+        Returns:
+            Optional[str]: Path to the markdown file or None if not found
+        """
+        if not os.path.isdir(directory):
+            return None
+
+        for file in os.listdir(directory):
+            if file.endswith(".md"):
+                return os.path.join(directory, file)
+        return None
+
+    @staticmethod
+    def parse_markdown_sections(content: str) -> List[Dict[str, Union[str, int, List]]]:
+        """
+        Parse markdown content and organize it by sections based on headers.
+
+        Args:
+            content: The markdown content to parse
+
+        Returns:
+            List[Dict]: A list of sections, each containing:
+                - level: The header level (1-6)
+                - title: The section title
+                - content: The section content
+                - subsections: List of subsections
+        """
+        # Split content into lines
+        lines = content.split("\n")
+        sections = []
+        current_section = None
+        current_content = []
+
+        for line in lines:
+            # Check if line is a header
+            header_match = re.match(r"^(#{1,6})\s+(.+)$", line)
+
+            if header_match:
+                # If we were building a section, save its content
+                if current_section is not None:
+                    current_section["content"] = "\n".join(current_content).strip()
+                    sections.append(current_section)
+
+                # Start a new section
+                level = len(header_match.group(1))
+                title = header_match.group(2).strip()
+                current_section = {
+                    "level": level,
+                    "title": title,
+                    "content": "",
+                    "subsections": [],
+                }
+                current_content = []
+            elif current_section is not None:
+                current_content.append(line)
+
+        # Don't forget to save the last section
+        if current_section is not None:
+            current_section["content"] = "\n".join(current_content).strip()
+            sections.append(current_section)
+
+        return FileProcessor._organize_sections(sections)
+
+    @staticmethod
+    def _organize_sections(sections: List[Dict]) -> List[Dict]:
+        """
+        Organize sections into a hierarchical structure based on their levels.
+
+        Args:
+            sections: List of sections with their levels
+
+        Returns:
+            List[Dict]: Organized hierarchical structure of sections
+        """
+        result = []
+        section_stack = []
+
+        for section in sections:
+            while section_stack and section_stack[-1]["level"] >= section["level"]:
+                section_stack.pop()
+
+            if section_stack:
+                section_stack[-1]["subsections"].append(section)
+            else:
+                result.append(section)
+
+            section_stack.append(section)
+
+        return result
+
+    @staticmethod
+    async def read_file_content(file_path: str) -> str:
+        """
+        Read the content of a file asynchronously.
+
+        Args:
+            file_path: Path to the file to read
+
+        Returns:
+            str: The content of the file
+
+        Raises:
+            FileNotFoundError: If the file doesn't exist
+            IOError: If there's an error reading the file
+        """
+        try:
+            # Ensure the file exists
+            if not os.path.exists(file_path):
+                raise FileNotFoundError(f"File not found: {file_path}")
+
+            # Read file content
+            # Note: Using async with would be better for large files
+            # but for simplicity and compatibility, using regular file reading
+            with open(file_path, "r", encoding="utf-8") as f:
+                content = f.read()
+
+            return content
+
+        except Exception as e:
+            raise IOError(f"Error reading file {file_path}: {str(e)}")
+
+    @staticmethod
+    def format_section_content(section: Dict) -> str:
+        """
+        Format a section's content with standardized spacing and structure.
+
+        Args:
+            section: Dictionary containing section information
+
+        Returns:
+            str: Formatted section content
+        """
+        # Start with section title
+        formatted = f"\n{'#' * section['level']} {section['title']}\n"
+
+        # Add section content if it exists
+        if section["content"]:
+            formatted += f"\n{section['content'].strip()}\n"
+
+        # Process subsections
+        if section["subsections"]:
+            # Add a separator before subsections if there's content
+            if section["content"]:
+                formatted += "\n---\n"
+
+            # Process each subsection
+            for subsection in section["subsections"]:
+                formatted += FileProcessor.format_section_content(subsection)
+
+        # Add section separator
+        formatted += "\n" + "=" * 80 + "\n"
+
+        return formatted
+
+    @staticmethod
+    def standardize_output(sections: List[Dict]) -> str:
+        """
+        Convert structured sections into a standardized string format.
+
+        Args:
+            sections: List of section dictionaries
+
+        Returns:
+            str: Standardized string output
+        """
+        output = []
+
+        # Process each top-level section
+        for section in sections:
+            output.append(FileProcessor.format_section_content(section))
+
+        # Join all sections with clear separation
+        return "\n".join(output)
+
+    @classmethod
+    async def process_file_input(
+        cls, file_input: Union[str, Dict], base_dir: str = None
+    ) -> Dict:
+        """
+        Process file input information and return the structured content.
+
+        Args:
+            file_input: File input information (JSON string, dict, or direct file path)
+            base_dir: Optional base directory to use for creating paper directories (for sync support)
+
+        Returns:
+            Dict: The structured content with sections and standardized text
+        """
+        try:
+            # 首先尝试从字符串中提取markdown文件路径
+            if isinstance(file_input, str):
+                import re
+
+                file_path_match = re.search(r"`([^`]+\.md)`", file_input)
+                if file_path_match:
+                    paper_path = file_path_match.group(1)
+                    file_input = {"paper_path": paper_path}
+
+            # Extract paper directory path
+            paper_dir = cls.extract_file_path(file_input)
+
+            # If base_dir is provided, adjust paper_dir to be relative to base_dir
+            if base_dir and paper_dir:
+                # If paper_dir is using default location, move it to base_dir
+                if paper_dir.endswith(("deepcode_lab", "agent_folders")):
+                    paper_dir = base_dir
+                else:
+                    # Extract the relative part and combine with base_dir
+                    paper_name = os.path.basename(paper_dir)
+                    # 保持原始目录名不变，不做任何替换
+                    paper_dir = os.path.join(base_dir, "papers", paper_name)
+
+                # Ensure the directory exists
+                os.makedirs(paper_dir, exist_ok=True)
+
+            if not paper_dir:
+                raise ValueError("Could not determine paper directory path")
+
+            # Get the actual file path
+            file_path = None
+            if isinstance(file_input, str):
+                # 尝试解析为JSON（处理下载结果）
+                try:
+                    parsed_json = json.loads(file_input)
+                    if isinstance(parsed_json, dict) and "paper_path" in parsed_json:
+                        file_path = parsed_json.get("paper_path")
+                        # 如果文件不存在，尝试查找markdown文件
+                        if file_path and not os.path.exists(file_path):
+                            paper_dir = os.path.dirname(file_path)
+                            if os.path.isdir(paper_dir):
+                                file_path = cls.find_markdown_file(paper_dir)
+                                if not file_path:
+                                    raise ValueError(
+                                        f"No markdown file found in directory: {paper_dir}"
+                                    )
+                    else:
+                        raise ValueError("Invalid JSON format: missing paper_path")
+                except json.JSONDecodeError:
+                    # 尝试从文本中提取JSON（处理包含额外文本的下载结果）
+                    extracted_json = cls.extract_json_from_text(file_input)
+                    if extracted_json and "paper_path" in extracted_json:
+                        file_path = extracted_json.get("paper_path")
+                        # 如果文件不存在，尝试查找markdown文件
+                        if file_path and not os.path.exists(file_path):
+                            paper_dir = os.path.dirname(file_path)
+                            if os.path.isdir(paper_dir):
+                                file_path = cls.find_markdown_file(paper_dir)
+                                if not file_path:
+                                    raise ValueError(
+                                        f"No markdown file found in directory: {paper_dir}"
+                                    )
+                    else:
+                        # 不是JSON，按文件路径处理
+                        # Check if it's a file path (existing or not)
+                        if file_input.endswith(
+                            (".md", ".pdf", ".txt", ".docx", ".doc", ".html", ".htm")
+                        ):
+                            if os.path.exists(file_input):
+                                file_path = file_input
+                            else:
+                                # File doesn't exist, try to find markdown in the directory
+                                file_path = cls.find_markdown_file(paper_dir)
+                                if not file_path:
+                                    raise ValueError(
+                                        f"No markdown file found in directory: {paper_dir}"
+                                    )
+                        elif os.path.exists(file_input):
+                            if os.path.isfile(file_input):
+                                file_path = file_input
+                            elif os.path.isdir(file_input):
+                                # If it's a directory, find the markdown file
+                                file_path = cls.find_markdown_file(file_input)
+                                if not file_path:
+                                    raise ValueError(
+                                        f"No markdown file found in directory: {file_input}"
+                                    )
+                        else:
+                            raise ValueError(f"Invalid input: {file_input}")
+            else:
+                # Dictionary input
+                file_path = file_input.get("paper_path")
+                # If the file doesn't exist, try to find markdown in the directory
+                if file_path and not os.path.exists(file_path):
+                    paper_dir = os.path.dirname(file_path)
+                    if os.path.isdir(paper_dir):
+                        file_path = cls.find_markdown_file(paper_dir)
+                        if not file_path:
+                            raise ValueError(
+                                f"No markdown file found in directory: {paper_dir}"
+                            )
+
+            if not file_path:
+                raise ValueError("No valid file path found")
+
+            # Read file content
+            content = await cls.read_file_content(file_path)
+
+            # Parse and structure the content
+            structured_content = cls.parse_markdown_sections(content)
+
+            # Generate standardized text output
+            standardized_text = cls.standardize_output(structured_content)
+
+            return {
+                "paper_dir": paper_dir,
+                "file_path": file_path,
+                "sections": structured_content,
+                "standardized_text": standardized_text,
+            }
+
+        except Exception as e:
+            raise ValueError(f"Error processing file input: {str(e)}")
+
+    @staticmethod
+    def extract_json_from_text(text: str) -> Optional[Dict]:
+        """
+        Extract JSON from text that may contain markdown code blocks or other content.
+
+        Args:
+            text: Text that may contain JSON
+
+        Returns:
+            Optional[Dict]: Extracted JSON as dictionary or None if not found
+        """
+        import re
+
+        # Try to find JSON in markdown code blocks
+        json_pattern = r"```json\s*(\{.*?\})\s*```"
+        match = re.search(json_pattern, text, re.DOTALL)
+        if match:
+            try:
+                return json.loads(match.group(1))
+            except json.JSONDecodeError:
+                pass
+
+        # Try to find standalone JSON
+        json_pattern = r"(\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\})"
+        matches = re.findall(json_pattern, text, re.DOTALL)
+        for match in matches:
+            try:
+                parsed = json.loads(match)
+                if isinstance(parsed, dict) and "paper_path" in parsed:
+                    return parsed
+            except json.JSONDecodeError:
+                continue
+
+        return None

utils/simple_llm_logger.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+超简化LLM响应日志记录器
+专注于记录LLM回复的核心内容，配置简单易用
+"""
+
+import json
+import os
+import yaml
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, Any
+
+
+class SimpleLLMLogger:
+    """超简化的LLM响应日志记录器"""
+
+    def __init__(self, config_path: str = "mcp_agent.config.yaml"):
+        """
+        初始化日志记录器
+
+        Args:
+            config_path: 配置文件路径
+        """
+        self.config = self._load_config(config_path)
+        self.llm_config = self.config.get("llm_logger", {})
+
+        # 如果禁用则直接返回
+        if not self.llm_config.get("enabled", True):
+            self.enabled = False
+            return
+
+        self.enabled = True
+        self._setup_logger()
+
+    def _load_config(self, config_path: str) -> Dict[str, Any]:
+        """加载配置文件"""
+        try:
+            with open(config_path, "r", encoding="utf-8") as f:
+                return yaml.safe_load(f)
+        except Exception as e:
+            print(f"⚠️ 配置文件加载失败: {e}，使用默认配置")
+            return self._get_default_config()
+
+    def _get_default_config(self) -> Dict[str, Any]:
+        """获取默认配置"""
+        return {
+            "llm_logger": {
+                "enabled": True,
+                "output_format": "json",
+                "log_level": "basic",
+                "log_directory": "logs/llm_responses",
+                "filename_pattern": "llm_responses_{timestamp}.jsonl",
+                "include_models": ["claude-sonnet-4", "gpt-4", "o3-mini"],
+                "min_response_length": 50,
+            }
+        }
+
+    def _setup_logger(self):
+        """设置日志记录器"""
+        log_dir = self.llm_config.get("log_directory", "logs/llm_responses")
+
+        # 创建日志目录
+        Path(log_dir).mkdir(parents=True, exist_ok=True)
+
+        # 生成日志文件名
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        filename_pattern = self.llm_config.get(
+            "filename_pattern", "llm_responses_{timestamp}.jsonl"
+        )
+        self.log_file = os.path.join(
+            log_dir, filename_pattern.format(timestamp=timestamp)
+        )
+
+        print(f"📝 LLM响应日志: {self.log_file}")
+
+    def log_response(self, content: str, model: str = "", agent: str = "", **kwargs):
+        """
+        记录LLM响应 - 简化版本
+
+        Args:
+            content: LLM响应内容
+            model: 模型名称
+            agent: Agent名称
+            **kwargs: 其他可选信息
+        """
+        if not self.enabled:
+            return
+
+        # 检查是否应该记录
+        if not self._should_log(content, model):
+            return
+
+        # 构建日志记录
+        log_entry = self._build_entry(content, model, agent, kwargs)
+
+        # 写入日志
+        self._write_log(log_entry)
+
+        # 控制台显示
+        self._console_log(content, model, agent)
+
+    def _should_log(self, content: str, model: str) -> bool:
+        """检查是否应该记录"""
+        # 检查长度
+        min_length = self.llm_config.get("min_response_length", 50)
+        if len(content) < min_length:
+            return False
+
+        # 检查模型
+        include_models = self.llm_config.get("include_models", [])
+        if include_models and not any(m in model for m in include_models):
+            return False
+
+        return True
+
+    def _build_entry(self, content: str, model: str, agent: str, extra: Dict) -> Dict:
+        """构建日志条目"""
+        log_level = self.llm_config.get("log_level", "basic")
+
+        if log_level == "basic":
+            # 基础级别：只记录核心内容
+            return {
+                "timestamp": datetime.now().isoformat(),
+                "content": content,
+                "model": model,
+            }
+        else:
+            # 详细级别：包含更多信息
+            entry = {
+                "timestamp": datetime.now().isoformat(),
+                "content": content,
+                "model": model,
+                "agent": agent,
+            }
+            # 添加额外信息
+            if "token_usage" in extra:
+                entry["tokens"] = extra["token_usage"]
+            if "session_id" in extra:
+                entry["session"] = extra["session_id"]
+            return entry
+
+    def _write_log(self, entry: Dict):
+        """写入日志文件"""
+        output_format = self.llm_config.get("output_format", "json")
+
+        try:
+            with open(self.log_file, "a", encoding="utf-8") as f:
+                if output_format == "json":
+                    f.write(json.dumps(entry, ensure_ascii=False) + "\n")
+                elif output_format == "text":
+                    timestamp = entry.get("timestamp", "")
+                    model = entry.get("model", "")
+                    content = entry.get("content", "")
+                    f.write(f"[{timestamp}] {model}: {content}\n\n")
+                elif output_format == "markdown":
+                    timestamp = entry.get("timestamp", "")
+                    model = entry.get("model", "")
+                    content = entry.get("content", "")
+                    f.write(f"**{timestamp}** | {model}\n\n{content}\n\n---\n\n")
+        except Exception as e:
+            print(f"⚠️ 写入日志失败: {e}")
+
+    def _console_log(self, content: str, model: str, agent: str):
+        """控制台简要显示"""
+        preview = content[:80] + "..." if len(content) > 80 else content
+        print(f"🤖 {model} ({agent}): {preview}")
+
+
+# 全局实例
+_global_logger = None
+
+
+def get_llm_logger() -> SimpleLLMLogger:
+    """获取全局LLM日志记录器实例"""
+    global _global_logger
+    if _global_logger is None:
+        _global_logger = SimpleLLMLogger()
+    return _global_logger
+
+
+def log_llm_response(content: str, model: str = "", agent: str = "", **kwargs):
+    """便捷函数：记录LLM响应"""
+    logger = get_llm_logger()
+    logger.log_response(content, model, agent, **kwargs)
+
+
+# 示例使用
+if __name__ == "__main__":
+    # 测试日志记录
+    log_llm_response(
+        content="这是一个测试的LLM响应内容，用于验证简化日志记录器的功能是否正常工作。",
+        model="claude-sonnet-4-20250514",
+        agent="TestAgent",
+    )
+
+    print("✅ 简化LLM日志测试完成")

workflows/__init__.py

@@ -0,0 +1,31 @@
+"""
+Intelligent Agent Orchestration Workflows for Research-to-Code Automation.
+
+This package provides advanced AI-driven workflow orchestration capabilities
+for automated research analysis and code implementation synthesis.
+"""
+
+from .agent_orchestration_engine import (
+    run_research_analyzer,
+    run_resource_processor,
+    run_code_analyzer,
+    github_repo_download,
+    paper_reference_analyzer,
+    execute_multi_agent_research_pipeline,
+    paper_code_preparation,  # Deprecated, for backward compatibility
+)
+
+from .code_implementation_workflow import CodeImplementationWorkflow
+
+__all__ = [
+    # Initial workflows
+    "run_research_analyzer",
+    "run_resource_processor",
+    "run_code_analyzer",
+    "github_repo_download",
+    "paper_reference_analyzer",
+    "execute_multi_agent_research_pipeline",  # Main multi-agent pipeline function
+    "paper_code_preparation",  # Deprecated, for backward compatibility
+    # Code implementation workflows
+    "CodeImplementationWorkflow",
+]