auto-coder 0.1.400__py3-none-any.whl → 2.0.0__py3-none-any.whl

autocoder/common/todos/models.py

@@ -0,0 +1,239 @@
+"""
+Todo manager data models.
+"""
+
+import time
+import uuid
+from typing import Union, Dict, List, Optional, Any
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Todo:
+    """Todo item data model"""
+    
+    content: str
+    todo_id: str = field(default_factory=lambda: str(uuid.uuid4())[:8])
+    status: str = "pending"
+    priority: str = "medium"
+    created_at: float = field(default_factory=time.time)
+    updated_at: float = field(default_factory=time.time)
+    notes: Optional[str] = None
+    dependencies: Optional[List[str]] = None
+    
+    def __post_init__(self):
+        """Data validation"""
+        self._validate()
+    
+    def _validate(self):
+        """Validate todo data"""
+        # Validate content
+        if not self.content or (isinstance(self.content, str) and len(self.content.strip()) == 0):
+            raise ValueError("Todo content cannot be empty")
+        
+        # Validate status
+        valid_statuses = ["pending", "in_progress", "completed", "cancelled"]
+        if self.status not in valid_statuses:
+            raise ValueError(f"Invalid status: {self.status}, valid statuses: {valid_statuses}")
+        
+        # Validate priority
+        valid_priorities = ["low", "medium", "high"]
+        if self.priority not in valid_priorities:
+            raise ValueError(f"Invalid priority: {self.priority}, valid priorities: {valid_priorities}")
+        
+        # Validate timestamps
+        if not isinstance(self.created_at, (int, float)) or self.created_at <= 0:
+            raise ValueError("Invalid created_at timestamp")
+        
+        if not isinstance(self.updated_at, (int, float)) or self.updated_at < self.created_at:
+            raise ValueError("Invalid updated_at timestamp")
+        
+        # Validate todo ID
+        if not isinstance(self.todo_id, str) or len(self.todo_id) == 0:
+            raise ValueError("Todo ID cannot be empty")
+    
+    def update_status(self, status: str):
+        """Update todo status"""
+        valid_statuses = ["pending", "in_progress", "completed", "cancelled"]
+        if status not in valid_statuses:
+            raise ValueError(f"Invalid status: {status}")
+        
+        self.status = status
+        self.updated_at = time.time()
+    
+    def update_content(self, content: str):
+        """Update todo content"""
+        if not content or len(content.strip()) == 0:
+            raise ValueError("Todo content cannot be empty")
+        
+        self.content = content
+        self.updated_at = time.time()
+    
+    def update_priority(self, priority: str):
+        """Update todo priority"""
+        valid_priorities = ["low", "medium", "high"]
+        if priority not in valid_priorities:
+            raise ValueError(f"Invalid priority: {priority}")
+        
+        self.priority = priority
+        self.updated_at = time.time()
+    
+    def add_notes(self, notes: str):
+        """Add or update notes"""
+        self.notes = notes
+        self.updated_at = time.time()
+    
+    def add_dependency(self, todo_id: str):
+        """Add a dependency to another todo"""
+        if self.dependencies is None:
+            self.dependencies = []
+        
+        if todo_id not in self.dependencies:
+            self.dependencies.append(todo_id)
+            self.updated_at = time.time()
+    
+    def remove_dependency(self, todo_id: str):
+        """Remove a dependency"""
+        if self.dependencies and todo_id in self.dependencies:
+            self.dependencies.remove(todo_id)
+            self.updated_at = time.time()
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Serialize to dictionary"""
+        return {
+            "todo_id": self.todo_id,
+            "content": self.content,
+            "status": self.status,
+            "priority": self.priority,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "notes": self.notes,
+            "dependencies": self.dependencies
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Todo":
+        """Deserialize from dictionary"""
+        return cls(
+            todo_id=data["todo_id"],
+            content=data["content"],
+            status=data.get("status", "pending"),
+            priority=data.get("priority", "medium"),
+            created_at=data["created_at"],
+            updated_at=data["updated_at"],
+            notes=data.get("notes"),
+            dependencies=data.get("dependencies")
+        )
+
+
+@dataclass
+class TodoList:
+    """Todo list data model for a conversation"""
+    
+    conversation_id: str
+    todos: List[Dict[str, Any]] = field(default_factory=list)
+    created_at: float = field(default_factory=time.time)
+    updated_at: float = field(default_factory=time.time)
+    metadata: Optional[Dict[str, Any]] = None
+    
+    def __post_init__(self):
+        """Data validation"""
+        self._validate()
+    
+    def _validate(self):
+        """Validate todo list data"""
+        # Validate conversation ID
+        if not isinstance(self.conversation_id, str) or len(self.conversation_id) == 0:
+            raise ValueError("Conversation ID cannot be empty")
+        
+        # Validate timestamps
+        if not isinstance(self.created_at, (int, float)) or self.created_at <= 0:
+            raise ValueError("Invalid created_at timestamp")
+        
+        if not isinstance(self.updated_at, (int, float)) or self.updated_at < self.created_at:
+            raise ValueError("Invalid updated_at timestamp")
+        
+        # Validate todos list
+        if not isinstance(self.todos, list):
+            raise ValueError("Todos must be a list")
+    
+    def add_todo(self, todo: Todo):
+        """Add a todo to the list"""
+        self.todos.append(todo.to_dict())
+        self.updated_at = time.time()
+    
+    def remove_todo(self, todo_id: str) -> bool:
+        """Remove a todo from the list"""
+        for i, todo_data in enumerate(self.todos):
+            if todo_data.get("todo_id") == todo_id:
+                del self.todos[i]
+                self.updated_at = time.time()
+                return True
+        return False
+    
+    def get_todo(self, todo_id: str) -> Optional[Dict[str, Any]]:
+        """Get a todo from the list"""
+        for todo_data in self.todos:
+            if todo_data.get("todo_id") == todo_id:
+                return todo_data
+        return None
+    
+    def update_todo(self, todo_id: str, **kwargs) -> bool:
+        """Update a todo in the list"""
+        for todo_data in self.todos:
+            if todo_data.get("todo_id") == todo_id:
+                # Update fields
+                for key, value in kwargs.items():
+                    if key in ["content", "status", "priority", "notes", "dependencies"]:
+                        todo_data[key] = value
+                
+                todo_data["updated_at"] = time.time()
+                self.updated_at = time.time()
+                return True
+        return False
+    
+    def get_todos_by_status(self, status: str) -> List[Dict[str, Any]]:
+        """Get todos by status"""
+        return [todo for todo in self.todos if todo.get("status") == status]
+    
+    def get_todos_by_priority(self, priority: str) -> List[Dict[str, Any]]:
+        """Get todos by priority"""
+        return [todo for todo in self.todos if todo.get("priority") == priority]
+    
+    def get_statistics(self) -> Dict[str, int]:
+        """Get todo statistics"""
+        stats = {
+            "total": len(self.todos),
+            "pending": 0,
+            "in_progress": 0,
+            "completed": 0,
+            "cancelled": 0
+        }
+        
+        for todo in self.todos:
+            status = todo.get("status", "pending")
+            if status in stats:
+                stats[status] += 1
+        
+        return stats
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Serialize to dictionary"""
+        return {
+            "conversation_id": self.conversation_id,
+            "todos": self.todos,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "metadata": self.metadata
+        }
+    
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "TodoList":
+        """Deserialize from dictionary"""
+        return cls(
+            conversation_id=data["conversation_id"],
+            todos=data.get("todos", []),
+            created_at=data["created_at"],
+            updated_at=data["updated_at"],
+            metadata=data.get("metadata")
+        ) 

autocoder/common/todos/storage/__init__.py

@@ -0,0 +1,14 @@
+"""
+Storage module for todo management.
+
+This module provides storage functionality for todos and todo lists,
+with support for file-based storage and extensible storage backends.
+"""
+
+from .base_storage import BaseStorage
+from .file_storage import FileStorage
+
+__all__ = [
+    'BaseStorage',
+    'FileStorage'
+] 

autocoder/common/todos/storage/base_storage.py

@@ -0,0 +1,76 @@
+"""
+Base storage interface for todo management.
+
+This module defines the abstract base class for all storage implementations,
+providing a consistent interface for todo and todo list operations.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Optional, Dict, Any, List
+
+
+class BaseStorage(ABC):
+    """Abstract base class for todo storage implementations."""
+    
+    @abstractmethod
+    def save_todo_list(self, conversation_id: str, todo_list_data: Dict[str, Any]) -> bool:
+        """
+        Save a todo list to storage.
+        
+        Args:
+            conversation_id: The conversation ID
+            todo_list_data: The todo list data to save
+            
+        Returns:
+            True if save was successful, False otherwise
+        """
+        pass
+    
+    @abstractmethod
+    def load_todo_list(self, conversation_id: str) -> Optional[Dict[str, Any]]:
+        """
+        Load a todo list from storage.
+        
+        Args:
+            conversation_id: The conversation ID
+            
+        Returns:
+            Todo list data or None if not found
+        """
+        pass
+    
+    @abstractmethod
+    def delete_todo_list(self, conversation_id: str) -> bool:
+        """
+        Delete a todo list from storage.
+        
+        Args:
+            conversation_id: The conversation ID
+            
+        Returns:
+            True if deletion was successful, False otherwise
+        """
+        pass
+    
+    @abstractmethod
+    def todo_list_exists(self, conversation_id: str) -> bool:
+        """
+        Check if a todo list exists in storage.
+        
+        Args:
+            conversation_id: The conversation ID
+            
+        Returns:
+            True if todo list exists, False otherwise
+        """
+        pass
+    
+    @abstractmethod
+    def list_conversation_ids(self) -> List[str]:
+        """
+        List all conversation IDs that have todo lists.
+        
+        Returns:
+            List of conversation IDs
+        """
+        pass 

autocoder/common/todos/storage/file_storage.py

@@ -0,0 +1,278 @@
+"""
+File-based storage implementation for todo lists.
+
+This module provides a file system storage implementation for todo lists,
+storing each conversation's todos as a separate JSON file.
+"""
+
+import os
+import json
+import tempfile
+from pathlib import Path
+from typing import Optional, Dict, Any, List
+from loguru import logger
+
+from .base_storage import BaseStorage
+
+
+class FileStorage(BaseStorage):
+    """File-based storage implementation for todo lists."""
+    
+    def __init__(self, storage_path: str):
+        """
+        Initialize file storage.
+        
+        Args:
+            storage_path: Base path for todo storage
+        """
+        self.storage_path = Path(storage_path)
+        self._ensure_storage_directory()
+    
+    def _ensure_storage_directory(self):
+        """Ensure storage directory exists."""
+        self.storage_path.mkdir(parents=True, exist_ok=True)
+    
+    def _get_todo_list_file_path(self, conversation_id: str) -> Path:
+        """
+        Get file path for a conversation's todo list.
+        
+        Args:
+            conversation_id: The conversation ID
+            
+        Returns:
+            Path to the todo list file
+        """
+        # Sanitize conversation ID for file system
+        safe_id = self._sanitize_filename(conversation_id)
+        return self.storage_path / f"{safe_id}.json"
+    
+    def _sanitize_filename(self, filename: str) -> str:
+        """
+        Sanitize filename to be safe for file system.
+        
+        Args:
+            filename: Original filename
+            
+        Returns:
+            Sanitized filename
+        """
+        # Replace unsafe characters with underscores
+        unsafe_chars = '<>:"/\\|?*'
+        safe_name = filename
+        for char in unsafe_chars:
+            safe_name = safe_name.replace(char, '_')
+        
+        # Limit length
+        if len(safe_name) > 200:
+            safe_name = safe_name[:200]
+        
+        return safe_name
+    
+    def _validate_todo_list_data(self, todo_list_data: Dict[str, Any]) -> bool:
+        """
+        Validate todo list data structure.
+        
+        Args:
+            todo_list_data: Todo list data to validate
+            
+        Returns:
+            True if valid, False otherwise
+        """
+        required_fields = ['conversation_id', 'todos', 'created_at', 'updated_at']
+        
+        for field in required_fields:
+            if field not in todo_list_data:
+                logger.error(f"Todo list validation failed: missing field '{field}'")
+                return False
+        
+        if not isinstance(todo_list_data['todos'], list):
+            logger.error("Todo list validation failed: 'todos' must be a list")
+            return False
+        
+        return True
+    
+    def _atomic_write_file(self, file_path: Path, data: Dict[str, Any]) -> bool:
+        """
+        Atomically write data to file.
+        
+        Args:
+            file_path: Target file path
+            data: Data to write
+            
+        Returns:
+            True if write was successful, False otherwise
+        """
+        try:
+            # Write to temporary file first
+            temp_fd, temp_path = tempfile.mkstemp(
+                suffix='.tmp',
+                dir=file_path.parent,
+                prefix=f'{file_path.stem}_'
+            )
+            
+            try:
+                with os.fdopen(temp_fd, 'w', encoding='utf-8') as f:
+                    json.dump(data, f, indent=2, ensure_ascii=False)
+                
+                # Atomic move
+                Path(temp_path).replace(file_path)
+                return True
+                
+            except Exception:
+                # Clean up temp file on error
+                try:
+                    os.unlink(temp_path)
+                except:
+                    pass
+                raise
+                
+        except Exception as e:
+            logger.error(f"Failed to write file atomically {file_path}: {e}")
+            return False
+    
+    def save_todo_list(self, conversation_id: str, todo_list_data: Dict[str, Any]) -> bool:
+        """
+        Save a todo list to storage.
+        
+        Args:
+            conversation_id: The conversation ID
+            todo_list_data: The todo list data to save
+            
+        Returns:
+            True if save was successful, False otherwise
+        """
+        try:
+            # Validate data
+            if not self._validate_todo_list_data(todo_list_data):
+                return False
+            
+            # Get file path
+            file_path = self._get_todo_list_file_path(conversation_id)
+            
+            # Atomic write
+            success = self._atomic_write_file(file_path, todo_list_data)
+            
+            if success:
+                logger.debug(f"Successfully saved todo list for conversation {conversation_id}")
+            else:
+                logger.error(f"Failed to save todo list for conversation {conversation_id}")
+            
+            return success
+            
+        except Exception as e:
+            logger.error(f"Error saving todo list for conversation {conversation_id}: {e}")
+            return False
+    
+    def load_todo_list(self, conversation_id: str) -> Optional[Dict[str, Any]]:
+        """
+        Load a todo list from storage.
+        
+        Args:
+            conversation_id: The conversation ID
+            
+        Returns:
+            Todo list data or None if not found
+        """
+        try:
+            file_path = self._get_todo_list_file_path(conversation_id)
+            
+            if not file_path.exists():
+                logger.debug(f"Todo list file not found for conversation {conversation_id}")
+                return None
+            
+            with open(file_path, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+            
+            # Validate loaded data
+            if not self._validate_todo_list_data(data):
+                logger.error(f"Invalid todo list data for conversation {conversation_id}")
+                return None
+            
+            logger.debug(f"Successfully loaded todo list for conversation {conversation_id}")
+            return data
+            
+        except json.JSONDecodeError as e:
+            logger.error(f"Invalid JSON in todo list file for conversation {conversation_id}: {e}")
+            return None
+        except Exception as e:
+            logger.error(f"Error loading todo list for conversation {conversation_id}: {e}")
+            return None
+    
+    def delete_todo_list(self, conversation_id: str) -> bool:
+        """
+        Delete a todo list from storage.
+        
+        Args:
+            conversation_id: The conversation ID
+            
+        Returns:
+            True if deletion was successful, False otherwise
+        """
+        try:
+            file_path = self._get_todo_list_file_path(conversation_id)
+            
+            if file_path.exists():
+                file_path.unlink()
+                logger.debug(f"Successfully deleted todo list for conversation {conversation_id}")
+                return True
+            else:
+                logger.debug(f"Todo list file not found for conversation {conversation_id} (already deleted)")
+                return True
+                
+        except Exception as e:
+            logger.error(f"Error deleting todo list for conversation {conversation_id}: {e}")
+            return False
+    
+    def todo_list_exists(self, conversation_id: str) -> bool:
+        """
+        Check if a todo list exists in storage.
+        
+        Args:
+            conversation_id: The conversation ID
+            
+        Returns:
+            True if todo list exists, False otherwise
+        """
+        try:
+            file_path = self._get_todo_list_file_path(conversation_id)
+            return file_path.exists()
+        except Exception as e:
+            logger.error(f"Error checking todo list existence for conversation {conversation_id}: {e}")
+            return False
+    
+    def list_conversation_ids(self) -> List[str]:
+        """
+        List all conversation IDs that have todo lists.
+        
+        Returns:
+            List of conversation IDs
+        """
+        try:
+            conversation_ids = []
+            
+            if not self.storage_path.exists():
+                return conversation_ids
+            
+            for file_path in self.storage_path.glob("*.json"):
+                # Extract conversation ID from filename
+                conversation_id = file_path.stem
+                
+                # Try to load and validate the file
+                try:
+                    with open(file_path, 'r', encoding='utf-8') as f:
+                        data = json.load(f)
+                    
+                    if self._validate_todo_list_data(data):
+                        # Use the conversation ID from the file content
+                        actual_id = data.get('conversation_id', conversation_id)
+                        conversation_ids.append(actual_id)
+                        
+                except Exception as e:
+                    logger.warning(f"Skipping invalid todo list file {file_path}: {e}")
+                    continue
+            
+            return conversation_ids
+            
+        except Exception as e:
+            logger.error(f"Error listing conversation IDs: {e}")
+            return [] 

autocoder/common/tokens/__init__.py

@@ -51,6 +51,20 @@ def count_directory_tokens(
     )
 
 
+def count_string_tokens(text: str) -> int:
+    """
+    统计字符串的 token 数量
+    
+    Args:
+        text: 要统计的字符串内容
+        
+    Returns:
+        int: token 数量
+    """
+    counter = TokenCounter()
+    return counter.count_string_tokens(text)
+
+
 __all__ = [
     'TokenResult',
     'DirectoryTokenResult',
@@ -59,4 +73,5 @@ __all__ = [
     'FileFilter',
     'count_file_tokens',
     'count_directory_tokens',
+    'count_string_tokens',
 ]

autocoder/common/tokens/counter.py

@@ -4,6 +4,7 @@ from typing import List, Dict, Optional, Union, Callable
 from pathlib import Path
 import time
 import re
+import importlib.resources as resources
 
 from autocoder.rag.variable_holder import VariableHolder
 from .models import TokenResult, DirectoryTokenResult
@@ -30,9 +31,30 @@ class TokenCounter:
         self.parallel = parallel
         self.max_workers = max_workers
         
-        # 确保 tokenizer 已经加载
+        # 确保 tokenizer 已经加载，如果没有则自动初始化
+        self._ensure_tokenizer_initialized()
+    
+    def _ensure_tokenizer_initialized(self):
+        """
+        确保 tokenizer 已经初始化，如果没有则自动初始化
+        """
         if VariableHolder.TOKENIZER_MODEL is None:
-            raise RuntimeError("Tokenizer model not initialized. Please call load_tokenizer() first.")
+            self._initialize_tokenizer()
+    
+    def _initialize_tokenizer(self):
+        """
+        初始化 tokenizer，参考 auto_coder_runner.py 中的 load_tokenizer 实现
+        """
+        try:
+            from tokenizers import Tokenizer
+            tokenizer_path = str(resources.files("autocoder") / "data" / "tokenizer.json")
+            VariableHolder.TOKENIZER_PATH = tokenizer_path
+            VariableHolder.TOKENIZER_MODEL = Tokenizer.from_file(tokenizer_path)
+        except (FileNotFoundError, ImportError) as e:
+            raise RuntimeError(
+                f"Failed to initialize tokenizer: {str(e)}. "
+                f"Make sure tokenizer.json exists in the autocoder package data directory."
+            )
     
     def count_file(self, file_path: str) -> TokenResult:
         """
@@ -200,6 +222,26 @@ class TokenCounter:
             errors=errors
         )
     
+    def count_string_tokens(self, text: str) -> int:
+        """
+        统计字符串的 token 数量
+        
+        Args:
+            text: 要统计的字符串内容
+            
+        Returns:
+            int: token 数量
+        """
+        try:
+            if not isinstance(text, str):
+                raise ValueError("Input must be a string")
+            
+            # 使用 tokenizer 编码字符串并统计 token 数量
+            tokens = VariableHolder.TOKENIZER_MODEL.encode(text)
+            return len(tokens)
+        except Exception as e:
+            raise RuntimeError(f"Failed to count tokens: {str(e)}")
+    
     def set_tokenizer(self, tokenizer_name: str) -> None:
         """
         更改 tokenizer（目前不支持，仅为接口预留）