maque 0.2.1__py3-none-any.whl

maque/cli/clean_invisible_chars.py

@@ -0,0 +1,324 @@
+#!/usr/bin/env python3
+"""
+文件不可见字符清理工具
+
+功能：
+- 清理文件中的不间断空格(U+00A0)和其他常见不可见字符
+- 支持单个文件或批量处理
+- 自动备份原文件
+- 提供详细的处理报告
+
+使用方法：
+    python clean_invisible_chars.py file.py
+    python clean_invisible_chars.py *.py
+    python clean_invisible_chars.py --dir /path/to/directory --pattern "*.py"
+"""
+
+import argparse
+import glob
+import os
+import shutil
+from pathlib import Path
+from typing import List, Tuple
+
+
+class InvisibleCharCleaner:
+    """不可见字符清理器"""
+
+    # 常见的需要清理的不可见字符映射
+    CHAR_REPLACEMENTS = {
+        "\u00a0": " ",  # 不间断空格 -> 普通空格
+        "\u2000": " ",  # en quad -> 普通空格
+        "\u2001": " ",  # em quad -> 普通空格
+        "\u2002": " ",  # en space -> 普通空格
+        "\u2003": " ",  # em space -> 普通空格
+        "\u2004": " ",  # three-per-em space -> 普通空格
+        "\u2005": " ",  # four-per-em space -> 普通空格
+        "\u2006": " ",  # six-per-em space -> 普通空格
+        "\u2007": " ",  # figure space -> 普通空格
+        "\u2008": " ",  # punctuation space -> 普通空格
+        "\u2009": " ",  # thin space -> 普通空格
+        "\u200a": " ",  # hair space -> 普通空格
+        "\u200b": "",  # 零宽空格 -> 删除
+        "\u200c": "",  # 零宽非连接符 -> 删除
+        "\u200d": "",  # 零宽连接符 -> 删除
+        "\u2060": "",  # 字间连接符 -> 删除
+        "\ufeff": "",  # 字节顺序标记(BOM) -> 删除
+    }
+
+    def __init__(self, backup=True, verbose=True):
+        """
+        初始化清理器
+
+        Args:
+            backup: 是否备份原文件
+            verbose: 是否显示详细信息
+        """
+        self.backup = backup
+        self.verbose = verbose
+        self.stats = {
+            "files_processed": 0,
+            "files_modified": 0,
+            "chars_replaced": 0,
+            "backup_created": 0,
+        }
+
+    def detect_invisible_chars(self, content: str) -> List[Tuple[str, int, str]]:
+        """
+        检测文件中的不可见字符
+
+        Args:
+            content: 文件内容
+
+        Returns:
+            检测到的不可见字符列表: [(字符, 数量, 描述)]
+        """
+        detected = []
+
+        char_descriptions = {
+            "\u00a0": "不间断空格",
+            "\u2000": "en quad",
+            "\u2001": "em quad",
+            "\u2002": "en space",
+            "\u2003": "em space",
+            "\u2004": "three-per-em space",
+            "\u2005": "four-per-em space",
+            "\u2006": "six-per-em space",
+            "\u2007": "figure space",
+            "\u2008": "punctuation space",
+            "\u2009": "thin space",
+            "\u200a": "hair space",
+            "\u200b": "零宽空格",
+            "\u200c": "零宽非连接符",
+            "\u200d": "零宽连接符",
+            "\u2060": "字间连接符",
+            "\ufeff": "字节顺序标记(BOM)",
+        }
+
+        for char, description in char_descriptions.items():
+            count = content.count(char)
+            if count > 0:
+                detected.append((char, count, description))
+
+        return detected
+
+    def clean_content(self, content: str) -> Tuple[str, int]:
+        """
+        清理文本内容中的不可见字符
+
+        Args:
+            content: 原始内容
+
+        Returns:
+            (清理后的内容, 替换的字符数量)
+        """
+        cleaned_content = content
+        total_replacements = 0
+
+        for old_char, new_char in self.CHAR_REPLACEMENTS.items():
+            count = cleaned_content.count(old_char)
+            if count > 0:
+                cleaned_content = cleaned_content.replace(old_char, new_char)
+                total_replacements += count
+
+        return cleaned_content, total_replacements
+
+    def backup_file(self, file_path: Path) -> Path:
+        """
+        备份文件
+
+        Args:
+            file_path: 原文件路径
+
+        Returns:
+            备份文件路径
+        """
+        backup_path = file_path.with_suffix(file_path.suffix + ".backup")
+        shutil.copy2(file_path, backup_path)
+        self.stats["backup_created"] += 1
+        return backup_path
+
+    def clean_file(self, file_path: Path) -> bool:
+        """
+        清理单个文件
+
+        Args:
+            file_path: 文件路径
+
+        Returns:
+            是否有修改
+        """
+        try:
+            # 读取文件
+            with open(file_path, "r", encoding="utf-8") as f:
+                original_content = f.read()
+
+            # 检测不可见字符
+            detected_chars = self.detect_invisible_chars(original_content)
+
+            if not detected_chars:
+                if self.verbose:
+                    print(f"✓ {file_path}: 未发现不可见字符")
+                return False
+
+            # 显示检测结果
+            if self.verbose:
+                print(f"\n📁 处理文件: {file_path}")
+                print("🔍 检测到的不可见字符:")
+                for char, count, desc in detected_chars:
+                    hex_code = f"U+{ord(char):04X}"
+                    print(f"  - {desc} ({hex_code}): {count} 个")
+
+            # 备份原文件
+            if self.backup:
+                backup_path = self.backup_file(file_path)
+                if self.verbose:
+                    print(f"💾 已备份到: {backup_path}")
+
+            # 清理内容
+            cleaned_content, replacements = self.clean_content(original_content)
+
+            # 写入清理后的内容
+            with open(file_path, "w", encoding="utf-8") as f:
+                f.write(cleaned_content)
+
+            # 更新统计
+            self.stats["files_modified"] += 1
+            self.stats["chars_replaced"] += replacements
+
+            if self.verbose:
+                print(f"✅ 已清理 {replacements} 个不可见字符")
+
+            return True
+
+        except Exception as e:
+            print(f"❌ 处理文件 {file_path} 时出错: {e}")
+            return False
+        finally:
+            self.stats["files_processed"] += 1
+
+    def clean_files(self, file_paths: List[Path]) -> None:
+        """
+        批量清理文件
+
+        Args:
+            file_paths: 文件路径列表
+        """
+        print(f"🚀 开始处理 {len(file_paths)} 个文件...")
+        print("=" * 60)
+
+        for file_path in file_paths:
+            if file_path.is_file():
+                self.clean_file(file_path)
+            else:
+                print(f"⚠️  跳过非文件: {file_path}")
+
+        # 显示统计信息
+        self.print_summary()
+
+    def print_summary(self) -> None:
+        """打印处理摘要"""
+        print("\n" + "=" * 60)
+        print("📊 处理摘要")
+        print("=" * 60)
+        print(f"处理文件数: {self.stats['files_processed']}")
+        print(f"修改文件数: {self.stats['files_modified']}")
+        print(f"清理字符数: {self.stats['chars_replaced']}")
+        if self.backup:
+            print(f"创建备份数: {self.stats['backup_created']}")
+        print("✨ 处理完成!")
+
+
+def find_files_by_pattern(directory: str, pattern: str) -> List[Path]:
+    """
+    根据模式查找文件
+
+    Args:
+        directory: 目录路径
+        pattern: 文件模式 (如 "*.py")
+
+    Returns:
+        匹配的文件路径列表
+    """
+    search_pattern = os.path.join(directory, "**", pattern)
+    file_paths = []
+
+    for path_str in glob.glob(search_pattern, recursive=True):
+        path = Path(path_str)
+        if path.is_file():
+            file_paths.append(path)
+
+    return sorted(file_paths)
+
+
+def main():
+    """主函数"""
+    parser = argparse.ArgumentParser(
+        description="清理文件中的不可见字符",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+使用示例:
+  %(prog)s file.py                     # 清理单个文件
+  %(prog)s *.py                        # 清理当前目录下所有Python文件
+  %(prog)s file1.py file2.py           # 清理多个文件
+  %(prog)s --dir /path --pattern "*.py" # 递归清理目录下的Python文件
+  %(prog)s --no-backup file.py         # 清理时不创建备份
+  %(prog)s --quiet file.py             # 静默模式
+        """,
+    )
+
+    parser.add_argument("files", nargs="*", help="要处理的文件路径")
+    parser.add_argument("--dir", "-d", help="要处理的目录路径")
+    parser.add_argument("--pattern", "-p", default="*", help='文件匹配模式 (如 "*.py")')
+    parser.add_argument("--no-backup", action="store_true", help="不创建备份文件")
+    parser.add_argument("--quiet", "-q", action="store_true", help="静默模式")
+
+    args = parser.parse_args()
+
+    # 收集要处理的文件
+    file_paths = []
+
+    if args.dir:
+        # 目录模式
+        if not os.path.isdir(args.dir):
+            print(f"❌ 目录不存在: {args.dir}")
+            return 1
+        file_paths = find_files_by_pattern(args.dir, args.pattern)
+        if not file_paths:
+            print(f"❌ 在目录 {args.dir} 中未找到匹配 {args.pattern} 的文件")
+            return 1
+    elif args.files:
+        # 文件列表模式
+        for file_pattern in args.files:
+            if "*" in file_pattern or "?" in file_pattern:
+                # 通配符模式
+                matched_files = glob.glob(file_pattern)
+                if matched_files:
+                    file_paths.extend([Path(f) for f in matched_files])
+                else:
+                    print(f"⚠️  未找到匹配 {file_pattern} 的文件")
+            else:
+                # 直接文件路径
+                file_path = Path(file_pattern)
+                if file_path.exists():
+                    file_paths.append(file_path)
+                else:
+                    print(f"⚠️  文件不存在: {file_pattern}")
+    else:
+        # 没有指定文件或目录
+        parser.print_help()
+        return 1
+
+    if not file_paths:
+        print("❌ 没有找到要处理的文件")
+        return 1
+
+    # 创建清理器并处理文件
+    cleaner = InvisibleCharCleaner(backup=not args.no_backup, verbose=not args.quiet)
+
+    cleaner.clean_files(file_paths)
+    return 0
+
+
+if __name__ == "__main__":
+    exit(main())

maque/cli/core.py

@@ -0,0 +1,34 @@
+import time
+import subprocess
+import io
+import csv
+import collections
+
+
+def run_job(job, num: int, interval="second"):
+    import schedule
+
+    if interval == "second":
+        schedule.every(num).seconds.do(job)
+    elif interval == "minute":
+        schedule.every(num).minutes.do(job)
+    elif interval == "hour":
+        schedule.every(num).hours.do(job)
+    elif interval == "day":
+        schedule.every(num).days.do(job)
+    while True:
+        schedule.run_pending()
+        time.sleep(1)
+
+
+def commandexists(shellcommand):
+    status, output = subprocess.getstatusoutput(shellcommand)
+    exists = status == 0
+    if not exists:
+        print("Could not execute: {0}".format(shellcommand))
+    return exists
+
+def command(args):
+    # subprocess.call(args)
+    # subprocess.getoutput(cmd)
+    return subprocess.check_output(args).decode()

maque/cli/groups/__init__.py

@@ -0,0 +1,26 @@
+# CLI命令组模块
+from .config import ConfigGroup
+from .mllm import MllmGroup
+from .data import DataGroup
+from .service import ServiceGroup
+from .doctor import DoctorGroup
+from .help import HelpGroup
+from .embedding import EmbeddingGroup
+from .git import GitGroup
+from .system import SystemGroup
+from .mcp import MCPGroup
+from .quant import QuantGroup
+
+__all__ = [
+    'ConfigGroup',
+    'MllmGroup',
+    'DataGroup',
+    'ServiceGroup',
+    'DoctorGroup',
+    'HelpGroup',
+    'EmbeddingGroup',
+    'GitGroup',
+    'SystemGroup',
+    'MCPGroup',
+    'QuantGroup',
+]

maque/cli/groups/config.py

@@ -0,0 +1,205 @@
+"""配置管理命令组"""
+import os
+import yaml
+from pathlib import Path
+from typing import Optional
+from rich.console import Console
+from rich.table import Table
+from rich import print
+
+
+class ConfigGroup:
+    """配置管理命令组"""
+    
+    def __init__(self, cli_instance):
+        self.cli = cli_instance
+        self.console = Console()
+    
+    def show(self):
+        """显示当前配置"""
+        config = self.cli.maque_config
+        
+        print("[bold blue]Sparrow 配置信息[/bold blue]\n")
+        
+        # 显示配置来源
+        config_paths = self.cli._get_config_search_paths()
+        for path in config_paths:
+            if path.exists():
+                print(f"[green]✓[/green] 配置文件: {path}")
+                break
+        else:
+            print("[yellow]⚠[/yellow] 未找到配置文件，使用默认配置")
+        
+        print("\n[bold]当前配置:[/bold]")
+        self._print_config_dict(config, indent=0)
+    
+    def _print_config_dict(self, config, indent=0):
+        """递归打印配置字典"""
+        prefix = "  " * indent
+        for key, value in config.items():
+            if isinstance(value, dict):
+                print(f"{prefix}[cyan]{key}[/cyan]:")
+                self._print_config_dict(value, indent + 1)
+            else:
+                print(f"{prefix}[cyan]{key}[/cyan]: [green]{value}[/green]")
+    
+    def edit(self, editor: str = None):
+        """交互式编辑配置文件
+        
+        Args:
+            editor: 使用的编辑器，默认使用系统默认编辑器
+        """
+        config_paths = self.cli._get_config_search_paths()
+        
+        # 寻找现有配置文件
+        config_file = None
+        for path in config_paths:
+            if path.exists():
+                config_file = path
+                break
+        
+        # 如果没有配置文件，创建一个
+        if not config_file:
+            config_file = self.cli._get_default_config_path()
+            if not config_file.exists():
+                self.cli.init_config()
+        
+        # 使用编辑器打开
+        editor_cmd = editor or os.environ.get('EDITOR', 'notepad' if os.name == 'nt' else 'nano')
+        os.system(f'{editor_cmd} "{config_file}"')
+        
+        print(f"[green]配置文件已编辑: {config_file}[/green]")
+        print("重新启动maque以应用新配置")
+    
+    def validate(self):
+        """验证配置文件"""
+        print("[bold blue]验证配置文件...[/bold blue]\n")
+        
+        config_paths = self.cli._get_config_search_paths()
+        errors = []
+        
+        for path in config_paths:
+            if path.exists():
+                try:
+                    with open(path, 'r', encoding='utf-8') as f:
+                        yaml.safe_load(f)
+                    print(f"[green]✓[/green] {path} - 语法正确")
+                    
+                    # 验证配置内容
+                    config = self.cli._load_maque_config()
+                    self._validate_config_content(config, errors)
+                    break
+                    
+                except yaml.YAMLError as e:
+                    errors.append(f"{path}: YAML语法错误 - {e}")
+                except Exception as e:
+                    errors.append(f"{path}: 读取错误 - {e}")
+        
+        if errors:
+            print("\n[bold red]发现配置错误:[/bold red]")
+            for error in errors:
+                print(f"[red]✗[/red] {error}")
+            return False
+        else:
+            print("\n[green]✓ 配置验证通过[/green]")
+            return True
+    
+    def _validate_config_content(self, config, errors):
+        """验证配置内容的合理性"""
+        # 验证MLLM配置
+        if 'mllm' in config:
+            mllm_config = config['mllm']
+            if 'base_url' in mllm_config:
+                url = mllm_config['base_url']
+                if not (url.startswith('http://') or url.startswith('https://')):
+                    errors.append("mllm.base_url 应该以 http:// 或 https:// 开头")
+    
+    def set(self, key: str, value: str, config_file: str = None):
+        """设置配置值
+
+        Args:
+            key: 配置键，支持点号分隔如 'mllm.model'
+            value: 配置值
+            config_file: 指定配置文件路径，默认使用 ~/.maque/config.yaml
+        """
+        if config_file is None:
+            config_path = self.cli._get_default_config_path()
+        else:
+            config_path = Path(config_file)
+        
+        # 加载现有配置
+        if config_path.exists():
+            with open(config_path, 'r', encoding='utf-8') as f:
+                config = yaml.safe_load(f) or {}
+        else:
+            config = {}
+        
+        # 设置嵌套键值
+        keys = key.split('.')
+        current = config
+        for k in keys[:-1]:
+            if k not in current:
+                current[k] = {}
+            current = current[k]
+        
+        # 尝试转换值的类型
+        try:
+            if value.lower() in ('true', 'false'):
+                value = value.lower() == 'true'
+            elif value.isdigit():
+                value = int(value)
+            elif value.replace('.', '').isdigit():
+                value = float(value)
+        except:
+            pass  # 保持字符串类型
+        
+        current[keys[-1]] = value
+        
+        # 保存配置
+        with open(config_path, 'w', encoding='utf-8') as f:
+            yaml.dump(config, f, default_flow_style=False, allow_unicode=True)
+        
+        print(f"[green]✓[/green] 已设置 {key} = {value}")
+        print(f"配置保存到: {config_path.resolve()}")
+    
+    def get(self, key: str = None):
+        """获取配置值
+        
+        Args:
+            key: 配置键，支持点号分隔如 'mllm.model'，为空则显示所有配置
+        """
+        if key is None:
+            return self.show()
+        
+        value = self.cli.get_config(key)
+        if value is not None:
+            print(f"[cyan]{key}[/cyan]: [green]{value}[/green]")
+        else:
+            print(f"[yellow]配置项 '{key}' 不存在[/yellow]")
+        
+        return value
+    
+    def reset(self, confirm: bool = False):
+        """重置配置到默认值
+        
+        Args:
+            confirm: 确认重置，为False时会提示确认
+        """
+        if not confirm:
+            response = input("确定要重置配置到默认值吗？这会覆盖现有配置 (y/N): ")
+            if response.lower() != 'y':
+                print("操作已取消")
+                return
+        
+        config_paths = self.cli._get_config_search_paths()
+        for path in config_paths:
+            if path.exists():
+                # 备份现有配置
+                backup_path = path.with_suffix(path.suffix + '.backup')
+                path.rename(backup_path)
+                print(f"原配置已备份到: {backup_path}")
+                break
+        
+        # 创建新的默认配置
+        self.cli.init_config()
+        print("[green]配置已重置为默认值[/green]")