auto-coder 0.1.396__py3-none-any.whl → 0.1.398__py3-none-any.whl

autocoder/common/conversations/storage/file_storage.py

@@ -0,0 +1,267 @@
+"""
+文件存储实现
+
+基于JSON文件的对话存储实现，支持原子写入和数据完整性检查。
+"""
+
+import os
+import json
+import tempfile
+import re
+from typing import Optional, List, Dict, Any
+from pathlib import Path
+
+from .base_storage import BaseStorage
+from ..exceptions import DataIntegrityError
+
+
+class FileStorage(BaseStorage):
+    """基于文件的存储实现"""
+    
+    def __init__(self, storage_path: str):
+        """
+        初始化文件存储
+        
+        Args:
+            storage_path: 存储目录路径
+        """
+        self.storage_path = Path(storage_path)
+        self._ensure_storage_directory()
+    
+    def _ensure_storage_directory(self):
+        """确保存储目录存在"""
+        self.storage_path.mkdir(parents=True, exist_ok=True)
+    
+    def _get_conversation_file_path(self, conversation_id: str) -> Path:
+        """
+        获取对话文件路径
+        
+        Args:
+            conversation_id: 对话ID
+            
+        Returns:
+            Path: 对话文件路径
+        """
+        # 清理文件名中的特殊字符
+        safe_filename = self._sanitize_filename(conversation_id)
+        return self.storage_path / f"{safe_filename}.json"
+    
+    def _sanitize_filename(self, filename: str) -> str:
+        """
+        清理文件名，移除或替换特殊字符
+        
+        Args:
+            filename: 原始文件名
+            
+        Returns:
+            str: 安全的文件名
+        """
+        # 移除或替换不安全的字符
+        safe_filename = re.sub(r'[<>:"/\\|?*]', '_', filename)
+        # 确保文件名不为空
+        if not safe_filename or safe_filename.isspace():
+            safe_filename = 'unnamed'
+        return safe_filename
+    
+    def _validate_conversation_data(self, conversation_data: Dict[str, Any]) -> bool:
+        """
+        验证对话数据的完整性
+        
+        Args:
+            conversation_data: 对话数据
+            
+        Returns:
+            bool: 数据有效返回True
+        """
+        if not isinstance(conversation_data, dict):
+            return False
+        
+        # 检查必需字段
+        required_fields = ['conversation_id']
+        for field in required_fields:
+            if field not in conversation_data:
+                return False
+            if not conversation_data[field]:
+                return False
+        
+        return True
+    
+    def _atomic_write_file(self, file_path: Path, data: Dict[str, Any]) -> bool:
+        """
+        原子写入文件
+        
+        Args:
+            file_path: 目标文件路径
+            data: 要写入的数据
+            
+        Returns:
+            bool: 写入成功返回True
+        """
+        temp_fd = None
+        temp_path = None
+        
+        try:
+            # 创建临时文件
+            temp_fd, temp_path = tempfile.mkstemp(
+                suffix='.tmp',
+                prefix=file_path.name + '.',
+                dir=file_path.parent
+            )
+            
+            # 写入数据到临时文件
+            with os.fdopen(temp_fd, 'w', encoding='utf-8') as temp_file:
+                temp_fd = None  # 文件已关闭，避免重复关闭
+                json.dump(data, temp_file, ensure_ascii=False, indent=2)
+            
+            # 原子重命名
+            os.rename(temp_path, file_path)
+            return True
+            
+        except (OSError, IOError, PermissionError, TypeError, ValueError):
+            # 清理临时文件
+            if temp_fd is not None:
+                try:
+                    os.close(temp_fd)
+                except OSError:
+                    pass
+            
+            if temp_path and os.path.exists(temp_path):
+                try:
+                    os.unlink(temp_path)
+                except OSError:
+                    pass
+            
+            return False
+    
+    def save_conversation(self, conversation_data: Dict[str, Any]) -> bool:
+        """
+        保存对话数据
+        
+        Args:
+            conversation_data: 对话数据字典
+            
+        Returns:
+            bool: 保存成功返回True
+        """
+        if not self._validate_conversation_data(conversation_data):
+            return False
+        
+        conversation_id = conversation_data['conversation_id']
+        file_path = self._get_conversation_file_path(conversation_id)
+        
+        return self._atomic_write_file(file_path, conversation_data)
+    
+    def load_conversation(self, conversation_id: str) -> Optional[Dict[str, Any]]:
+        """
+        加载对话数据
+        
+        Args:
+            conversation_id: 对话ID
+            
+        Returns:
+            Optional[Dict[str, Any]]: 对话数据，不存在返回None
+        """
+        file_path = self._get_conversation_file_path(conversation_id)
+        
+        if not file_path.exists():
+            return None
+        
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+            
+            # 验证加载的数据
+            if not self._validate_conversation_data(data):
+                raise DataIntegrityError(f"对话数据无效: {conversation_id}")
+            
+            return data
+            
+        except (json.JSONDecodeError, UnicodeDecodeError) as e:
+            raise DataIntegrityError(f"对话文件损坏: {conversation_id}, 错误: {str(e)}")
+        except (OSError, IOError) as e:
+            # 文件读取错误，返回None
+            return None
+    
+    def delete_conversation(self, conversation_id: str) -> bool:
+        """
+        删除对话数据
+        
+        Args:
+            conversation_id: 对话ID
+            
+        Returns:
+            bool: 删除成功返回True
+        """
+        file_path = self._get_conversation_file_path(conversation_id)
+        
+        if not file_path.exists():
+            return False
+        
+        try:
+            file_path.unlink()
+            return True
+        except (OSError, IOError):
+            return False
+    
+    def conversation_exists(self, conversation_id: str) -> bool:
+        """
+        检查对话是否存在
+        
+        Args:
+            conversation_id: 对话ID
+            
+        Returns:
+            bool: 存在返回True
+        """
+        file_path = self._get_conversation_file_path(conversation_id)
+        return file_path.exists()
+    
+    def list_conversations(
+        self, 
+        limit: Optional[int] = None, 
+        offset: int = 0
+    ) -> List[Dict[str, Any]]:
+        """
+        列出对话
+        
+        Args:
+            limit: 限制返回数量
+            offset: 偏移量
+            
+        Returns:
+            List[Dict[str, Any]]: 对话数据列表
+        """
+        conversations = []
+        
+        try:
+            # 获取所有JSON文件
+            json_files = list(self.storage_path.glob("*.json"))
+            
+            # 按修改时间排序（最新的在前）
+            json_files.sort(key=lambda x: x.stat().st_mtime, reverse=True)
+            
+            # 应用偏移量和限制
+            if limit is not None:
+                json_files = json_files[offset:offset + limit]
+            else:
+                json_files = json_files[offset:]
+            
+            # 加载对话数据
+            for file_path in json_files:
+                try:
+                    with open(file_path, 'r', encoding='utf-8') as f:
+                        data = json.load(f)
+                    
+                    # 验证数据
+                    if self._validate_conversation_data(data):
+                        conversations.append(data)
+                        
+                except (json.JSONDecodeError, UnicodeDecodeError, OSError, IOError):
+                    # 跳过损坏的文件
+                    continue
+                    
+        except OSError:
+            # 目录访问错误，返回空列表
+            pass
+        
+        return conversations 

autocoder/common/conversations/storage/index_manager.py

@@ -0,0 +1,317 @@
+"""
+索引管理器实现
+
+提供对话索引管理功能，支持快速查询、搜索和过滤。
+"""
+
+import os
+import json
+import time
+from typing import Optional, List, Dict, Any
+from pathlib import Path
+
+from ..file_locker import FileLocker
+from ..exceptions import DataIntegrityError
+
+
+class IndexManager:
+    """索引管理器，用于管理对话索引"""
+    
+    def __init__(self, index_path: str):
+        """
+        初始化索引管理器
+        
+        Args:
+            index_path: 索引目录路径
+        """
+        self.index_path = Path(index_path)
+        self.index_file = self.index_path / "conversations.idx"
+        self.lock_file = self.index_path / "index.lock"
+        
+        self._ensure_index_directory()
+        self._load_index()
+    
+    def _ensure_index_directory(self):
+        """确保索引目录存在"""
+        self.index_path.mkdir(parents=True, exist_ok=True)
+    
+    def _load_index(self):
+        """加载索引数据"""
+        try:
+            if self.index_file.exists():
+                with open(self.index_file, 'r', encoding='utf-8') as f:
+                    self._index_data = json.load(f)
+            else:
+                self._index_data = {}
+        except (json.JSONDecodeError, OSError, IOError):
+            # 如果索引损坏，重建空索引
+            self._index_data = {}
+    
+    def _save_index(self) -> bool:
+        """
+        保存索引数据
+        
+        Returns:
+            bool: 保存成功返回True
+        """
+        try:
+            # 使用临时文件进行原子写入
+            temp_file = self.index_file.with_suffix('.tmp')
+            
+            with open(temp_file, 'w', encoding='utf-8') as f:
+                json.dump(self._index_data, f, ensure_ascii=False, indent=2)
+            
+            # 原子重命名
+            temp_file.replace(self.index_file)
+            return True
+            
+        except (OSError, IOError):
+            return False
+    
+    def add_conversation(self, conversation_metadata: Dict[str, Any]) -> bool:
+        """
+        添加对话到索引
+        
+        Args:
+            conversation_metadata: 对话元数据
+            
+        Returns:
+            bool: 添加成功返回True
+        """
+        if not conversation_metadata.get('conversation_id'):
+            return False
+        
+        conversation_id = conversation_metadata['conversation_id']
+        
+        try:
+            # 重新加载索引以获取最新数据
+            self._load_index()
+            
+            # 添加或更新对话元数据
+            self._index_data[conversation_id] = conversation_metadata.copy()
+            
+            # 保存索引
+            return self._save_index()
+            
+        except Exception:
+            return False
+    
+    def update_conversation(self, conversation_metadata: Dict[str, Any]) -> bool:
+        """
+        更新索引中的对话
+        
+        Args:
+            conversation_metadata: 对话元数据
+            
+        Returns:
+            bool: 更新成功返回True
+        """
+        if not conversation_metadata.get('conversation_id'):
+            return False
+        
+        conversation_id = conversation_metadata['conversation_id']
+        
+        try:
+            # 重新加载索引以获取最新数据
+            self._load_index()
+            
+            # 检查对话是否存在
+            if conversation_id not in self._index_data:
+                return False
+            
+            # 更新对话元数据
+            self._index_data[conversation_id] = conversation_metadata.copy()
+            
+            # 保存索引
+            return self._save_index()
+            
+        except Exception:
+            return False
+    
+    def remove_conversation(self, conversation_id: str) -> bool:
+        """
+        从索引中删除对话
+        
+        Args:
+            conversation_id: 对话ID
+            
+        Returns:
+            bool: 删除成功返回True
+        """
+        try:
+            # 重新加载索引以获取最新数据
+            self._load_index()
+            
+            # 检查对话是否存在
+            if conversation_id not in self._index_data:
+                return False
+            
+            # 删除对话
+            del self._index_data[conversation_id]
+            
+            # 保存索引
+            return self._save_index()
+            
+        except Exception:
+            return False
+    
+    def get_conversation(self, conversation_id: str) -> Optional[Dict[str, Any]]:
+        """
+        从索引获取对话信息
+        
+        Args:
+            conversation_id: 对话ID
+            
+        Returns:
+            Optional[Dict[str, Any]]: 对话元数据，不存在返回None
+        """
+        try:
+            # 重新加载索引以获取最新数据
+            self._load_index()
+            
+            return self._index_data.get(conversation_id)
+            
+        except Exception:
+            return None
+    
+    def conversation_exists(self, conversation_id: str) -> bool:
+        """
+        检查对话是否在索引中存在
+        
+        Args:
+            conversation_id: 对话ID
+            
+        Returns:
+            bool: 存在返回True
+        """
+        return self.get_conversation(conversation_id) is not None
+    
+    def list_conversations(
+        self,
+        limit: Optional[int] = None,
+        offset: int = 0,
+        sort_by: str = 'updated_at',
+        sort_order: str = 'desc'
+    ) -> List[Dict[str, Any]]:
+        """
+        列出对话
+        
+        Args:
+            limit: 限制返回数量
+            offset: 偏移量
+            sort_by: 排序字段
+            sort_order: 排序顺序 ('asc' 或 'desc')
+            
+        Returns:
+            List[Dict[str, Any]]: 对话元数据列表
+        """
+        try:
+            # 重新加载索引以获取最新数据
+            self._load_index()
+            
+            # 获取所有对话
+            conversations = list(self._index_data.values())
+            
+            # 排序
+            reverse = (sort_order.lower() == 'desc')
+            
+            if sort_by == 'name':
+                conversations.sort(
+                    key=lambda x: x.get('name', ''),
+                    reverse=reverse
+                )
+            elif sort_by == 'created_at':
+                conversations.sort(
+                    key=lambda x: x.get('created_at', 0),
+                    reverse=reverse
+                )
+            elif sort_by == 'updated_at':
+                conversations.sort(
+                    key=lambda x: x.get('updated_at', 0),
+                    reverse=reverse
+                )
+            
+            # 应用分页
+            if limit is not None:
+                return conversations[offset:offset + limit]
+            else:
+                return conversations[offset:]
+                
+        except Exception:
+            return []
+    
+    def search_conversations(
+        self,
+        query: Optional[str] = None,
+        filters: Optional[Dict[str, Any]] = None
+    ) -> List[Dict[str, Any]]:
+        """
+        搜索对话
+        
+        Args:
+            query: 搜索查询字符串
+            filters: 过滤条件
+            
+        Returns:
+            List[Dict[str, Any]]: 匹配的对话元数据列表
+        """
+        try:
+            # 重新加载索引以获取最新数据
+            self._load_index()
+            
+            conversations = list(self._index_data.values())
+            results = []
+            
+            for conv in conversations:
+                # 文本搜索
+                if query:
+                    query_lower = query.lower()
+                    name_match = query_lower in conv.get('name', '').lower()
+                    desc_match = query_lower in conv.get('description', '').lower()
+                    
+                    if not (name_match or desc_match):
+                        continue
+                
+                # 应用过滤器
+                if filters:
+                    match = True
+                    
+                    # 时间范围过滤
+                    # created_after: 大于等于这个时间的记录
+                    if 'created_after' in filters:
+                        created_at = conv.get('created_at', 0)
+                        if created_at < filters['created_after']:
+                            match = False
+                    
+                    # created_before: 小于这个时间的记录（不包含边界）
+                    if 'created_before' in filters:
+                        created_at = conv.get('created_at', float('inf'))
+                        if created_at >= filters['created_before']:
+                            match = False
+                    
+                    # 消息数量过滤
+                    if 'min_message_count' in filters:
+                        message_count = conv.get('message_count', 0)
+                        if message_count < filters['min_message_count']:
+                            match = False
+                    
+                    if 'max_message_count' in filters:
+                        message_count = conv.get('message_count', float('inf'))
+                        if message_count > filters['max_message_count']:
+                            match = False
+                    
+                    if not match:
+                        continue
+                
+                results.append(conv)
+            
+            # 按相关性或更新时间排序
+            results.sort(
+                key=lambda x: x.get('updated_at', 0),
+                reverse=True
+            )
+            
+            return results
+            
+        except Exception:
+            return [] 

autocoder/common/rag_manager/rag_manager.py

@@ -79,24 +79,22 @@ class RAGManager:
             with open(config_path, 'r', encoding='utf-8') as f:
                 config_data = json.load(f)
             
-            for key, item in config_data.items():
-                try:
-                    # 构造 server_name: http://host:port/v1
-                    host = item.get("host", "127.0.0.1")
-                    port = item.get("port", 8080)
-                    server_name = f"http://{host}:{port}/v1"
-                    
-                    rag_config = RAGConfig(
-                        name=item.get("name", key),
-                        server_name=server_name,
-                        api_key=None,  # 全局配置中没有 api_key
-                        description=item.get("description", f"{key} RAG 服务")
-                    )
-                    self.configs.append(rag_config)
-                    logger.info(f"已加载 RAG 配置: {rag_config.name} -> {rag_config.server_name}")
-                except Exception as e:
-                    logger.error(f"解析全局 RAG 配置项时出错: {e}, 配置项: {item}")
-                    
+            if "data" in config_data and isinstance(config_data["data"], list):
+                for item in config_data["data"]:
+                    try:
+                        rag_config = RAGConfig(
+                            name=item.get("name", ""),
+                            server_name=item.get("base_url", ""),
+                            api_key=item.get("api_key"),
+                            description=item.get("description")
+                        )
+                        self.configs.append(rag_config)
+                        logger.info(f"已加载 RAG 配置: {rag_config.name} -> {rag_config.server_name}")
+                    except Exception as e:
+                        logger.error(f"解析全局 RAG 配置项时出错: {e}, 配置项: {item}")
+            else:
+                logger.error(f"全局 RAG 配置格式错误，缺少 'data' 字段或 'data' 不是列表")
+                
         except json.JSONDecodeError as e:
             logger.error(f"全局 RAG 配置文件 JSON 格式错误: {e}")
         except Exception as e: