auto-coder 0.1.400__py3-none-any.whl → 2.0.0__py3-none-any.whl

autocoder/plugins/token_helper_plugin.py

@@ -9,9 +9,10 @@ from dataclasses import dataclass
 from collections import defaultdict
 
 from autocoder.plugins import Plugin, PluginManager
-from autocoder.rag.token_counter import count_tokens
+from autocoder.common.tokens import count_string_tokens as count_tokens
 from autocoder.suffixproject import SuffixProject
 from autocoder.common import AutoCoderArgs, SourceCode
+from autocoder.common.international import get_message, get_message_with_format
 from loguru import logger
 import json
 
@@ -56,7 +57,7 @@ class TokenHelperPlugin(Plugin):
             True if initialization was successful
         """
         self.load_auto_coder_config()
-        print(f"[{self.name}] Token Helper plugin initialized")
+        print(f"[{self.name}] {get_message('plugin_token_initialized')}")
         return True
 
     def get_commands(self) -> Dict[str, Tuple[Callable, str]]:
@@ -66,10 +67,10 @@ class TokenHelperPlugin(Plugin):
             A dictionary of command name to handler and description
         """
         return {
-            "token/count": (self.count_tokens_in_project, "Count tokens in all project files"),
-            "token/top": (self.show_top_token_files, "Show top N files by token count"),
-            "token/file": (self.count_tokens_in_file, "Count tokens in a specific file"),
-            "token/summary": (self.show_token_summary, "Show token count summary for the project"),
+            "token/count": (self.count_tokens_in_project, get_message("plugin_token_cmd_count_desc")),
+            "token/top": (self.show_top_token_files, get_message("plugin_token_cmd_top_desc")),
+            "token/file": (self.count_tokens_in_file, get_message("plugin_token_cmd_file_desc")),
+            "token/summary": (self.show_token_summary, get_message("plugin_token_cmd_summary_desc")),
         }
 
     def get_completions(self) -> Dict[str, List[str]]:
@@ -98,8 +99,8 @@ class TokenHelperPlugin(Plugin):
         exclude = args_dict.get("exclude_files", [])
         
         self.project_dir = project_dir
-        print(f"Counting tokens in project: {project_dir}")
-        print(f"File types: {project_type}")
+        print(get_message_with_format("plugin_token_counting_project", project_dir=project_dir))
+        print(get_message_with_format("plugin_token_file_types", project_type=project_type))
         
         try:
             # Create AutoCoderArgs with necessary parameters
@@ -116,12 +117,12 @@ class TokenHelperPlugin(Plugin):
             file_count = 0
             self.token_counts = {}
             
-            print("Scanning files and counting tokens...")
+            print(get_message("plugin_token_scanning_files"))
             
             for source_code in project.get_source_codes():
                 file_count += 1
                 if file_count % 10 == 0:
-                    print(f"Processed {file_count} files...")
+                    print(get_message_with_format("plugin_token_processed_files", file_count=file_count))
                 
                 tokens = count_tokens(source_code.source_code)
                 file_path = source_code.module_name
@@ -137,15 +138,15 @@ class TokenHelperPlugin(Plugin):
                 
                 total_tokens += tokens
             
-            print(f"\nToken count complete!")
-            print(f"Total files: {file_count}")
-            print(f"Total tokens: {total_tokens:,}")
-            print(f"Use /token/top N to see the top N files by token count")
-            print(f"Use /token/summary to see a summary by file type")
+            print(f"\n{get_message('plugin_token_count_complete')}")
+            print(get_message_with_format("plugin_token_total_files", file_count=file_count))
+            print(get_message_with_format("plugin_token_total_tokens", total_tokens=f"{total_tokens:,}"))
+            print(get_message("plugin_token_use_top_help"))
+            print(get_message("plugin_token_use_summary_help"))
             
         except Exception as e:
             logger.error(f"Error counting tokens in project: {str(e)}")
-            print(f"Error: {str(e)}")
+            print(get_message_with_format("plugin_token_counting_error", error=str(e)))
 
     def _parse_args(self, args: str) -> Dict[str, str]:
         """Parse command arguments.
@@ -250,17 +251,22 @@ class TokenHelperPlugin(Plugin):
             args: Number of files to show
         """
         if not self.token_counts:
-            print("No token count data available. Run /token/count first.")
+            print(get_message("plugin_token_no_data"))
             return
             
         try:
             n = int(args.strip()) if args.strip() else 10
         except ValueError:
-            print(f"Invalid value: {args}. Using default of 10.")
+            print(get_message_with_format("plugin_token_invalid_value_default", args=args))
             n = 10
             
-        print(f"\nTop {n} files by token count:")
-        print(f"{'Tokens':<10} {'Size (bytes)':<15} {'File'}")
+        print(f"\n{get_message_with_format('plugin_token_top_files_header', n=n)}")
+        
+        tokens_header = get_message("plugin_token_table_header_tokens")
+        size_header = get_message("plugin_token_table_header_size_bytes") 
+        file_header = get_message("plugin_token_table_header_file")
+        
+        print(f"{tokens_header:<10} {size_header:<15} {file_header}")
         print(f"{'-'*10} {'-'*15} {'-'*50}")
         
         sorted_files = sorted(
@@ -274,29 +280,135 @@ class TokenHelperPlugin(Plugin):
             print(f"{token_count.tokens:<10,} {token_count.file_size:<15,} {relative_path}")
 
     def count_tokens_in_file(self, args: str) -> None:
-        """Count tokens in a specific file.
+        """Count tokens in a specific file or directory.
         
         Args:
-            args: Path to the file
+            args: Path to the file or directory. If starts with @, remove @ and treat as path.
         """
         if not args:
-            print("Please specify a file path.")
+            print(get_message("plugin_token_specify_path"))
             return
             
-        file_path = args.strip()
+        # Handle @ prefix - remove it and treat as path
+        path = args.strip()
+        if path.startswith('@'):
+            path = path[1:]
         
+        if not os.path.exists(path):
+            print(get_message_with_format("plugin_token_path_not_exist", path=path))
+            return
+            
+        try:
+            if os.path.isfile(path):
+                # Handle single file
+                self._count_tokens_single_file(path)
+            elif os.path.isdir(path):
+                # Handle directory recursively
+                self._count_tokens_directory(path)
+            else:
+                print(get_message_with_format("plugin_token_not_file_or_dir", path=path))
+                
+        except Exception as e:
+            print(get_message_with_format("plugin_token_counting_error", error=str(e)))
+    
+    def _count_tokens_single_file(self, file_path: str) -> int:
+        """Count tokens in a single file and display results.
+        
+        Args:
+            file_path: Path to the file
+            
+        Returns:
+            Number of tokens in the file
+        """
         try:
             with open(file_path, 'r', encoding='utf-8') as f:
                 content = f.read()
                 
             tokens = count_tokens(content)
-            print(f"\nFile: {file_path}")
-            print(f"Tokens: {tokens:,}")
-            print(f"File size: {len(content):,} bytes")
-            print(f"Avg bytes per token: {len(content)/tokens:.2f}")
+            print(f"\n{get_message_with_format('plugin_token_file_info', file_path=file_path)}")
+            print(get_message_with_format("plugin_token_file_tokens", tokens=f"{tokens:,}"))
+            print(get_message_with_format("plugin_token_file_size", size=f"{len(content):,}"))
+            if tokens > 0:
+                avg_bytes = len(content)/tokens
+                print(get_message_with_format("plugin_token_avg_bytes_per_token", avg=f"{avg_bytes:.2f}"))
+            
+            return tokens
             
+        except UnicodeDecodeError:
+            print(get_message_with_format("plugin_token_skip_binary", file_path=file_path))
+            return 0
         except Exception as e:
-            print(f"Error counting tokens in file: {str(e)}")
+            print(get_message_with_format("plugin_token_read_error", file_path=file_path, error=str(e)))
+            return 0
+    
+    def _count_tokens_directory(self, dir_path: str) -> None:
+        """Count tokens in all files within a directory recursively.
+        
+        Args:
+            dir_path: Path to the directory
+        """
+        total_tokens = 0
+        file_count = 0
+        processed_files = []
+        
+        print(f"\n{get_message_with_format('plugin_token_scanning_directory', dir_path=dir_path)}")
+        
+        for root, dirs, files in os.walk(dir_path):
+            # Skip common ignore directories
+            dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['__pycache__', 'node_modules', 'dist', 'build']]
+            
+            for file in files:
+                # Skip hidden files and common binary/generated files
+                if file.startswith('.') or file.endswith(('.pyc', '.pyo', '.so', '.dll', '.exe', '.bin')):
+                    continue
+                    
+                file_path = os.path.join(root, file)
+                relative_path = os.path.relpath(file_path, dir_path)
+                
+                try:
+                    with open(file_path, 'r', encoding='utf-8') as f:
+                        content = f.read()
+                        
+                    tokens = count_tokens(content)
+                    total_tokens += tokens
+                    file_count += 1
+                    
+                    processed_files.append({
+                        'path': relative_path,
+                        'tokens': tokens,
+                        'size': len(content)
+                    })
+                    
+                except (UnicodeDecodeError, PermissionError):
+                    # Skip binary files and files without permission
+                    continue
+                except Exception as e:
+                    print(get_message_with_format("plugin_token_processing_error", relative_path=relative_path, error=str(e)))
+                    continue
+        
+        # Display results
+        print(f"\n{get_message('plugin_token_scan_complete')}")
+        print(get_message_with_format("plugin_token_files_processed", file_count=file_count))
+        print(get_message_with_format("plugin_token_total_tokens", total_tokens=f"{total_tokens:,}"))
+        
+        if file_count > 0:
+            avg_tokens = total_tokens / file_count
+            print(get_message_with_format("plugin_token_avg_tokens_per_file", avg_tokens=f"{avg_tokens:.2f}"))
+            
+            # Show top 10 files by token count
+            if len(processed_files) > 1:
+                print(f"\n{get_message('plugin_token_top_files_by_count')}")
+                sorted_files = sorted(processed_files, key=lambda x: x['tokens'], reverse=True)
+                
+                tokens_header = get_message("plugin_token_table_header_tokens")
+                size_header = get_message("plugin_token_table_header_size")
+                file_header = get_message("plugin_token_table_header_file")
+                
+                print(f"{tokens_header:>8} {size_header:>8} {file_header}")
+                print(f"{'-'*8} {'-'*8} {'-'*50}")
+                
+                for file_info in sorted_files[:10]:
+                    print(f"{file_info['tokens']:>8,} {file_info['size']:>8} {file_info['path']}")
 
     def show_token_summary(self, args: str) -> None:
         """Show token count summary by file type.
@@ -305,7 +417,7 @@ class TokenHelperPlugin(Plugin):
             args: Optional arguments
         """
         if not self.token_counts:
-            print("No token count data available. Run /token/count first.")
+            print(get_message("plugin_token_no_data"))
             return
             
         by_extension = defaultdict(lambda: {"files": 0, "tokens": 0, "size": 0})
@@ -322,8 +434,15 @@ class TokenHelperPlugin(Plugin):
         total_files = sum(data["files"] for data in by_extension.values())
         total_size = sum(data["size"] for data in by_extension.values())
         
-        print(f"\nToken count summary by file type:")
-        print(f"{'Extension':<12} {'Files':<8} {'Tokens':<12} {'% of Total':<12} {'Size (KB)':<12}")
+        print(f"\n{get_message('plugin_token_summary_header')}")
+        
+        extension_header = get_message("plugin_token_table_header_extension")
+        files_header = get_message("plugin_token_table_header_files")
+        tokens_header = get_message("plugin_token_table_header_tokens")
+        percent_header = get_message("plugin_token_table_header_percent")
+        size_kb_header = get_message("plugin_token_table_header_size_kb")
+        
+        print(f"{extension_header:<12} {files_header:<8} {tokens_header:<12} {percent_header:<12} {size_kb_header:<12}")
         print(f"{'-'*12} {'-'*8} {'-'*12} {'-'*12} {'-'*12}")
         
         for ext, data in sorted(by_extension.items(), key=lambda x: x[1]["tokens"], reverse=True):
@@ -331,13 +450,13 @@ class TokenHelperPlugin(Plugin):
             size_kb = data["size"] / 1024
             print(f"{ext:<12} {data['files']:<8} {data['tokens']:<12,} {percent:<12.2f} {size_kb:<12.2f}")
             
-        print(f"\nTotal Files: {total_files:,}")
-        print(f"Total Tokens: {total_tokens:,}")
-        print(f"Total Size: {total_size/1024/1024:.2f} MB")
+        print(f"\n{get_message_with_format('plugin_token_total_files_summary', total_files=f'{total_files:,}')}")
+        print(get_message_with_format("plugin_token_total_tokens_summary", total_tokens=f"{total_tokens:,}"))
+        print(get_message_with_format("plugin_token_total_size", total_size=f"{total_size/1024/1024:.2f}"))
         
         if self.project_dir:
-            print(f"Project Directory: {self.project_dir}")
+            print(get_message_with_format("plugin_token_project_directory", project_dir=self.project_dir))
 
     def shutdown(self) -> None:
         """Shutdown the plugin."""
-        print(f"[{self.name}] Token Helper plugin shutdown") 
+        print(f"[{self.name}] {get_message('plugin_token_shutdown')}") 

autocoder/pyproject/__init__.py

@@ -15,10 +15,9 @@ from loguru import logger
 import re
 from pydantic import BaseModel, Field
 from rich.console import Console
-import json
-from autocoder.utils.queue_communicate import queue_communicate, CommunicateEvent, CommunicateEventType
 from autocoder.common import files as FileUtils
 
+
 class RegPattern(BaseModel):
     pattern: str = Field(
         ...,
@@ -169,12 +168,12 @@ class PyProject:
 
     def should_exclude(self, file_path):
         for pattern in self.exclude_patterns:
-            if pattern.search(file_path):                
+            if pattern.search(file_path):
                 return True
         return False
 
     def output(self):
-        with open(self.target_file, "r",encoding="utf-8") as file:
+        with open(self.target_file, "r", encoding="utf-8") as file:
             return file.read()
 
     def is_python_file(self, file_path):
@@ -183,7 +182,9 @@ class PyProject:
     def read_file_content(self, file_path):
         if self.args.auto_merge == "strict_diff":
             result = []
-            for line_number, line in FileUtils.read_file_with_line_numbers(file_path,line_number_start=1):
+            for line_number, line in FileUtils.read_file_with_line_numbers(
+                file_path, line_number_start=1
+            ):
                 result.append(f"{line_number}:{line}")
             return "\n".join(result)
 
@@ -233,20 +234,25 @@ class PyProject:
     def get_rag_source_codes(self):
         if not self.args.enable_rag_search and not self.args.enable_rag_context:
             return []
-                    
+
         else:
             console = Console()
-            console.print(f"\n[bold blue]Starting RAG search for:[/bold blue] {self.args.query}")
-            
+            console.print(
+                f"\n[bold blue]Starting RAG search for:[/bold blue] {self.args.query}"
+            )
+
         from autocoder.rag.rag_entry import RAGFactory
+
         rag = RAGFactory.get_rag(self.llm, self.args, "")
         docs = rag.search(self.args.query)
         for doc in docs:
-            doc.tag = "RAG"                    
+            doc.tag = "RAG"
         else:
             console = Console()
-            console.print(f"[bold green]Found {len(docs)} relevant documents[/bold green]")
-            
+            console.print(
+                f"[bold green]Found {len(docs)} relevant documents[/bold green]"
+            )
+
         return docs
 
     def get_search_source_codes(self):
@@ -275,7 +281,7 @@ class PyProject:
         return temp + []
 
     def get_source_codes(self) -> Generator[SourceCode, None, None]:
-        for root, dirs, files in os.walk(self.directory,followlinks=True):
+        for root, dirs, files in os.walk(self.directory, followlinks=True):
             dirs[:] = [d for d in dirs if d not in self.default_exclude_dirs]
             for file in files:
                 file_path = os.path.join(root, file)
@@ -339,27 +345,24 @@ class PyProject:
         if self.git_url is not None:
             self.clone_repository()
 
-        if self.target_file:
-            with open(self.target_file, "w",encoding="utf-8") as file:
+        # 先收集所有sources
+        for code in self.get_rest_source_codes():
+            self.sources.append(code)
 
-                for code in self.get_rest_source_codes():
-                    self.sources.append(code)                    
-                    file.write(f"##File: {code.module_name}\n")
-                    file.write(f"{code.source_code}\n\n")
+        for code in self.get_search_source_codes():
+            self.sources.append(code)
 
-                for code in self.get_search_source_codes():                    
-                    self.sources.append(code)                    
-                    file.write(f"##File: {code.module_name}\n")
-                    file.write(f"{code.source_code}\n\n")
+        for package in packages:
+            for code in self.get_package_source_codes(package):
+                self.sources.append(code)
 
-                for package in packages:
-                    for code in self.get_package_source_codes(package):
-                        self.sources.append(code)
-                        file.write(f"##File: {code.module_name}\n")
-                        file.write(f"{code.source_code}\n\n")
+        for code in self.get_source_codes():
+            self.sources.append(code)
 
-                for code in self.get_source_codes():                    
-                    self.sources.append(code)
+        # 如果target_file存在，才写入文件
+        if self.target_file:
+            with open(self.target_file, "w", encoding="utf-8") as file:
+                for code in self.sources:
                     file.write(f"##File: {code.module_name}\n")
                     file.write(f"{code.source_code}\n\n")