auto-coder 0.1.399__py3-none-any.whl → 1.0.0__py3-none-any.whl

autocoder/common/pull_requests/utils.py

@@ -0,0 +1,344 @@
+"""
+Pull Request 模块工具函数
+"""
+import re
+import os
+import subprocess
+from typing import Optional, Tuple
+from urllib.parse import urlparse
+from pathlib import Path
+from loguru import logger
+
+from .models import PlatformType, RepoInfo
+from .exceptions import ValidationError
+
+
+def parse_git_url(url: str) -> Tuple[Optional[PlatformType], Optional[str], Optional[str]]:
+    """
+    解析Git URL，提取平台类型、所有者和仓库名
+    
+    Returns:
+        Tuple[平台类型, 所有者, 仓库名]
+    """
+    if not url:
+        return None, None, None
+    
+    platform_domains = {
+        'github.com': PlatformType.GITHUB,
+        'gitlab.com': PlatformType.GITLAB,
+        'gitee.com': PlatformType.GITEE,
+        'gitcode.net': PlatformType.GITCODE
+    }
+    
+    # SSH URL 格式: git@domain:owner/repo.git
+    ssh_pattern = r'^git@([^:]+):([^/]+)/([^/]+?)(?:\.git)?/?$'
+    ssh_match = re.match(ssh_pattern, url)
+    
+    if ssh_match:
+        domain, owner, repo = ssh_match.groups()
+        platform = platform_domains.get(domain)
+        return platform, owner, repo
+    
+    # HTTPS URL 格式: https://domain/owner/repo.git
+    try:
+        parsed = urlparse(url)
+        domain = parsed.netloc
+        platform = platform_domains.get(domain)
+        
+        if not platform:
+            return None, None, None
+        
+        path_parts = [p for p in parsed.path.split('/') if p]
+        if len(path_parts) >= 2:
+            owner = path_parts[0]
+            repo = path_parts[1]
+            if repo.endswith('.git'):
+                repo = repo[:-4]
+            return platform, owner, repo
+    
+    except Exception as e:
+        logger.error(f"解析Git URL失败: {e}")
+    
+    return None, None, None
+
+
+def get_repo_remote_url(repo_path: str, remote_name: str = 'origin') -> Optional[str]:
+    """获取仓库的远程URL"""
+    try:
+        result = subprocess.run(
+            ['git', 'remote', 'get-url', remote_name],
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        return result.stdout.strip()
+    except subprocess.CalledProcessError:
+        return None
+
+
+def get_default_remote_branch(repo_path: str, remote_name: str = 'origin') -> Optional[str]:
+    """
+    获取默认远程分支
+    
+    Args:
+        repo_path: 仓库路径
+        remote_name: 远程名称，默认为 'origin'
+        
+    Returns:
+        默认远程分支名，如果获取失败则返回 None
+    """
+    try:
+        # 首先尝试获取远程的 HEAD 指向的分支
+        result = subprocess.run(
+            ['git', 'symbolic-ref', f'refs/remotes/{remote_name}/HEAD'],
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        # 输出格式通常是 "refs/remotes/origin/main"，我们需要提取分支名
+        head_ref = result.stdout.strip()
+        if head_ref.startswith(f'refs/remotes/{remote_name}/'):
+            return head_ref[len(f'refs/remotes/{remote_name}/'):]
+    except subprocess.CalledProcessError:
+        # 如果上面的方法失败，尝试从远程获取 HEAD 信息
+        try:
+            result = subprocess.run(
+                ['git', 'ls-remote', '--symref', remote_name, 'HEAD'],
+                cwd=repo_path,
+                capture_output=True,
+                text=True,
+                check=True
+            )
+            # 解析输出，查找类似 "ref: refs/heads/main	HEAD"
+            lines = result.stdout.strip().split('\n')
+            for line in lines:
+                if line.startswith('ref: refs/heads/'):
+                    return line.split('refs/heads/')[-1].split('\t')[0]
+        except subprocess.CalledProcessError:
+            pass
+    
+    # 如果都失败了，检查常见的默认分支是否存在
+    common_branches = ['main', 'master', 'develop']
+    for branch in common_branches:
+        if branch_exists(repo_path, branch, remote=True):
+            return branch
+    
+    return None
+
+
+def detect_platform_from_repo(repo_path: str) -> Optional[PlatformType]:
+    """从仓库路径自动检测平台类型"""
+    remote_url = get_repo_remote_url(repo_path)
+    if not remote_url:
+        return None
+    
+    platform, _, _ = parse_git_url(remote_url)
+    return platform
+
+
+def get_repo_info_from_path(repo_path: str) -> Optional[RepoInfo]:
+    """从仓库路径获取仓库信息"""
+    remote_url = get_repo_remote_url(repo_path)
+    if not remote_url:
+        return None
+    
+    platform, owner, name = parse_git_url(remote_url)
+    if not all([platform, owner, name]):
+        return None
+    
+    return RepoInfo(
+        platform=platform,  # type: ignore
+        owner=owner,  # type: ignore
+        name=name  # type: ignore
+    )
+
+
+def get_current_branch(repo_path: str) -> Optional[str]:
+    """获取当前分支名"""
+    try:
+        result = subprocess.run(
+            ['git', 'branch', '--show-current'],
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        return result.stdout.strip()
+    except subprocess.CalledProcessError:
+        return None
+
+
+def branch_exists(repo_path: str, branch_name: str, remote: bool = False) -> bool:
+    """检查分支是否存在"""
+    try:
+        if remote:
+            result = subprocess.run(
+                ['git', 'ls-remote', '--heads', 'origin', branch_name],
+                cwd=repo_path,
+                capture_output=True,
+                text=True,
+                check=True
+            )
+            return bool(result.stdout.strip())
+        else:
+            result = subprocess.run(
+                ['git', 'show-ref', '--verify', '--quiet', f'refs/heads/{branch_name}'],
+                cwd=repo_path,
+                capture_output=True
+            )
+            return result.returncode == 0
+    except subprocess.CalledProcessError:
+        return False
+
+
+def is_git_repo(path: str) -> bool:
+    """检查路径是否为Git仓库"""
+    git_dir = Path(path) / '.git'
+    return git_dir.exists() or git_dir.is_file()
+
+
+def push_branch_to_remote(repo_path: str, branch_name: str, remote_name: str = 'origin') -> bool:
+    """
+    推送分支到远程仓库
+    
+    Args:
+        repo_path: 仓库路径
+        branch_name: 分支名称
+        remote_name: 远程名称，默认为 'origin'
+        
+    Returns:
+        推送是否成功
+    """
+    try:
+        logger.info(f"正在推送分支 '{branch_name}' 到远程仓库...")
+        result = subprocess.run(
+            ['git', 'push', remote_name, branch_name],
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        logger.info(f"分支 '{branch_name}' 推送成功")
+        return True
+    except subprocess.CalledProcessError as e:
+        logger.error(f"推送分支 '{branch_name}' 失败: {e.stderr}")
+        return False
+
+
+def ensure_branch_exists_remotely(repo_path: str, branch_name: str, remote_name: str = 'origin') -> bool:
+    """
+    确保分支在远程仓库中存在，如果不存在则推送
+    
+    Args:
+        repo_path: 仓库路径
+        branch_name: 分支名称
+        remote_name: 远程名称，默认为 'origin'
+        
+    Returns:
+        分支是否存在于远程仓库（推送后）
+    """
+    # 首先检查分支是否已经存在于远程
+    if branch_exists(repo_path, branch_name, remote=True):
+        logger.debug(f"分支 '{branch_name}' 已存在于远程仓库")
+        return True
+    
+    # 检查分支是否存在于本地
+    if not branch_exists(repo_path, branch_name, remote=False):
+        logger.error(f"分支 '{branch_name}' 在本地也不存在")
+        return False
+    
+    # 推送分支到远程
+    return push_branch_to_remote(repo_path, branch_name, remote_name)
+
+
+def is_main_branch(branch_name: str) -> bool:
+    """
+    检查是否为主分支
+    
+    Args:
+        branch_name: 分支名称
+        
+    Returns:
+        是否为主分支
+    """
+    main_branches = ['main', 'master', 'develop', 'dev']
+    return branch_name.lower() in main_branches
+
+
+def create_and_checkout_branch(repo_path: str, branch_name: str) -> bool:
+    """
+    创建并切换到新分支
+    
+    Args:
+        repo_path: 仓库路径
+        branch_name: 新分支名称
+        
+    Returns:
+        操作是否成功
+    """
+    try:
+        logger.info(f"正在创建并切换到新分支: {branch_name}")
+        result = subprocess.run(
+            ['git', 'checkout', '-b', branch_name],
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            check=True
+        )
+        logger.info(f"成功创建并切换到分支: {branch_name}")
+        return True
+    except subprocess.CalledProcessError as e:
+        logger.error(f"创建分支 '{branch_name}' 失败: {e.stderr}")
+        return False
+
+
+def generate_auto_branch_name() -> str:
+    """
+    生成自动分支名称，格式为 ac-<yyyyMMdd-HH-mm-ss>
+    
+    Returns:
+        分支名称
+    """
+    import datetime
+    now = datetime.datetime.now()
+    return f"ac-{now.strftime('%Y%m%d-%H-%M-%S')}"
+
+
+def validate_repo_path(repo_path: str) -> str:
+    """验证并规范化仓库路径"""
+    if not repo_path:
+        raise ValidationError("仓库路径不能为空")
+    
+    path = Path(repo_path).resolve()
+    
+    if not path.exists():
+        raise ValidationError(f"仓库路径不存在: {path}")
+    
+    if not path.is_dir():
+        raise ValidationError(f"仓库路径不是目录: {path}")
+    
+    if not is_git_repo(str(path)):
+        raise ValidationError(f"路径不是Git仓库: {path}")
+    
+    return str(path)
+
+
+def build_pr_url(platform: PlatformType, repo_info: RepoInfo, pr_number: int) -> str:
+    """构建PR的Web URL"""
+    base_urls = {
+        PlatformType.GITHUB: "https://github.com",
+        PlatformType.GITLAB: "https://gitlab.com",
+        PlatformType.GITEE: "https://gitee.com",
+        PlatformType.GITCODE: "https://gitcode.net"
+    }
+    
+    base_url = base_urls.get(platform)
+    if not base_url:
+        return ""
+    
+    if platform == PlatformType.GITLAB or platform == PlatformType.GITCODE:
+        return f"{base_url}/{repo_info.full_name}/-/merge_requests/{pr_number}"
+    else:
+        return f"{base_url}/{repo_info.full_name}/pull/{pr_number}"

autocoder/common/tokens/__init__.py

@@ -0,0 +1,77 @@
+"""
+Tokens 模块 - 高效统计文件和目录中的 token 数量
+
+提供了简单易用的接口，支持正则过滤和智能文件类型识别。
+"""
+
+from .models import TokenResult, DirectoryTokenResult
+from .counter import TokenCounter
+from .file_detector import FileTypeDetector
+from .filters import FileFilter
+
+
+def count_file_tokens(file_path: str) -> TokenResult:
+    """
+    统计单个文件的 token 数量
+    
+    Args:
+        file_path: 文件路径
+        
+    Returns:
+        TokenResult: 统计结果
+    """
+    counter = TokenCounter()
+    return counter.count_file(file_path)
+
+
+def count_directory_tokens(
+    dir_path: str, 
+    pattern: str = None,
+    exclude_pattern: str = None,
+    recursive: bool = True
+) -> DirectoryTokenResult:
+    """
+    统计目录中所有文件的 token 数量
+    
+    Args:
+        dir_path: 目录路径
+        pattern: 文件名匹配模式（正则表达式）
+        exclude_pattern: 排除的文件名模式（正则表达式）
+        recursive: 是否递归处理子目录
+        
+    Returns:
+        DirectoryTokenResult: 目录统计结果
+    """
+    counter = TokenCounter()
+    return counter.count_directory(
+        dir_path=dir_path,
+        pattern=pattern,
+        exclude_pattern=exclude_pattern,
+        recursive=recursive
+    )
+
+
+def count_string_tokens(text: str) -> int:
+    """
+    统计字符串的 token 数量
+    
+    Args:
+        text: 要统计的字符串内容
+        
+    Returns:
+        int: token 数量
+    """
+    counter = TokenCounter()
+    return counter.count_string_tokens(text)
+
+
+__all__ = [
+    'TokenResult',
+    'DirectoryTokenResult',
+    'TokenCounter',
+    'FileTypeDetector',
+    'FileFilter',
+    'count_file_tokens',
+    'count_directory_tokens',
+    'count_string_tokens',
+]

autocoder/common/tokens/counter.py

@@ -0,0 +1,231 @@
+import os
+import concurrent.futures
+from typing import List, Dict, Optional, Union, Callable
+from pathlib import Path
+import time
+import re
+
+from autocoder.rag.variable_holder import VariableHolder
+from .models import TokenResult, DirectoryTokenResult
+from .file_detector import FileTypeDetector
+from .filters import FileFilter
+
+
+class TokenCounter:
+    """Token 计数器，用于统计文件和目录的 token 数量"""
+    
+    def __init__(self, 
+                 timeout: int = 30, 
+                 parallel: bool = True, 
+                 max_workers: int = 4):
+        """
+        初始化 Token 计数器
+        
+        Args:
+            timeout: 单文件处理超时时间（秒）
+            parallel: 是否并行处理
+            max_workers: 最大工作线程数
+        """
+        self.timeout = timeout
+        self.parallel = parallel
+        self.max_workers = max_workers
+        
+        # 确保 tokenizer 已经加载
+        if VariableHolder.TOKENIZER_MODEL is None:
+            raise RuntimeError("Tokenizer model not initialized. Please call load_tokenizer() first.")
+    
+    def count_file(self, file_path: str) -> TokenResult:
+        """
+        统计单个文件的 token 数量
+        
+        Args:
+            file_path: 文件路径
+            
+        Returns:
+            TokenResult: 统计结果
+        """
+        try:
+            if not os.path.isfile(file_path):
+                return TokenResult(
+                    file_path=file_path,
+                    token_count=0,
+                    char_count=0,
+                    line_count=0,
+                    success=False,
+                    error="File does not exist"
+                )
+            
+            # 检查是否为文本文件
+            if not FileTypeDetector.is_text_file(file_path):
+                return TokenResult(
+                    file_path=file_path,
+                    token_count=0,
+                    char_count=0,
+                    line_count=0,
+                    success=False,
+                    error="Not a text file"
+                )
+            
+            # 检测文件编码
+            encoding = FileTypeDetector.detect_encoding(file_path)
+            
+            # 读取文件内容
+            with open(file_path, 'r', encoding=encoding, errors='replace') as f:
+                content = f.read()
+            
+            # 统计行数
+            line_count = content.count('\n') + (0 if content == "" or content.endswith('\n') else 1)
+            
+            # 统计字符数
+            char_count = len(content)
+            
+            # 统计 token 数量
+            tokens = VariableHolder.TOKENIZER_MODEL.encode(content)
+            token_count = len(tokens)
+            
+            return TokenResult(
+                file_path=file_path,
+                token_count=token_count,
+                char_count=char_count,
+                line_count=line_count
+            )
+        except Exception as e:
+            return TokenResult(
+                file_path=file_path,
+                token_count=0,
+                char_count=0,
+                line_count=0,
+                success=False,
+                error=str(e)
+            )
+    
+    def count_files(self, file_paths: List[str]) -> List[TokenResult]:
+        """
+        批量统计多个文件的 token 数量
+        
+        Args:
+            file_paths: 文件路径列表
+            
+        Returns:
+            List[TokenResult]: 统计结果列表
+        """
+        if not self.parallel or len(file_paths) <= 1:
+            return [self.count_file(file_path) for file_path in file_paths]
+        
+        results = []
+        with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            future_to_file = {
+                executor.submit(self.count_file, file_path): file_path 
+                for file_path in file_paths
+            }
+            
+            for future in concurrent.futures.as_completed(future_to_file):
+                results.append(future.result())
+                
+        return results
+    
+    def count_directory(self, 
+                        dir_path: str, 
+                        pattern: str = None,
+                        exclude_pattern: str = None,
+                        recursive: bool = True,
+                        max_depth: int = None) -> DirectoryTokenResult:
+        """
+        统计目录中所有文件的 token 数量
+        
+        Args:
+            dir_path: 目录路径
+            pattern: 文件名匹配模式（正则表达式）
+            exclude_pattern: 排除的文件名模式（正则表达式）
+            recursive: 是否递归处理子目录
+            max_depth: 最大递归深度
+            
+        Returns:
+            DirectoryTokenResult: 目录统计结果
+        """
+        if not os.path.isdir(dir_path):
+            return DirectoryTokenResult(
+                directory_path=dir_path,
+                total_tokens=0,
+                file_count=0,
+                skipped_count=0,
+                files=[],
+                errors=["Directory does not exist"]
+            )
+        
+        # 创建文件过滤器
+        patterns = [pattern] if pattern else []
+        exclude_patterns = [exclude_pattern] if exclude_pattern else []
+        file_filter = FileFilter(patterns=patterns, exclude_patterns=exclude_patterns)
+        
+        # 收集所有匹配的文件
+        all_files = []
+        skipped_count = 0
+        
+        for root, dirs, files in os.walk(dir_path):
+            # 检查递归深度
+            if max_depth is not None:
+                current_depth = root[len(dir_path):].count(os.sep)
+                if current_depth >= max_depth:
+                    dirs.clear()  # 不再递归子目录
+            
+            for file in files:
+                file_path = os.path.join(root, file)
+                if file_filter.matches(file_path):
+                    all_files.append(file_path)
+                else:
+                    skipped_count += 1
+            
+            if not recursive:
+                break  # 不递归处理子目录
+        
+        # 统计所有文件
+        file_results = self.count_files(all_files)
+        
+        # 计算总 token 数
+        total_tokens = sum(result.token_count for result in file_results if result.success)
+        
+        # 收集错误
+        errors = [
+            f"{result.file_path}: {result.error}" 
+            for result in file_results if not result.success
+        ]
+        
+        return DirectoryTokenResult(
+            directory_path=dir_path,
+            total_tokens=total_tokens,
+            file_count=len(file_results),
+            skipped_count=skipped_count,
+            files=file_results,
+            errors=errors
+        )
+    
+    def count_string_tokens(self, text: str) -> int:
+        """
+        统计字符串的 token 数量
+        
+        Args:
+            text: 要统计的字符串内容
+            
+        Returns:
+            int: token 数量
+        """
+        try:
+            if not isinstance(text, str):
+                raise ValueError("Input must be a string")
+            
+            # 使用 tokenizer 编码字符串并统计 token 数量
+            tokens = VariableHolder.TOKENIZER_MODEL.encode(text)
+            return len(tokens)
+        except Exception as e:
+            raise RuntimeError(f"Failed to count tokens: {str(e)}")
+    
+    def set_tokenizer(self, tokenizer_name: str) -> None:
+        """
+        更改 tokenizer（目前不支持，仅为接口预留）
+        
+        Args:
+            tokenizer_name: tokenizer 名称
+        """
+        # 目前仅支持默认的 tokenizer
+        pass