rosetta-cli 2.0.0__py3-none-any.whl

rosetta_cli/__init__.py

@@ -0,0 +1,12 @@
+"""Rosetta CLI package."""
+
+from __future__ import annotations
+
+try:
+    from importlib.metadata import version
+
+    __version__ = version("rosetta-cli")
+except Exception:
+    __version__ = "unknown"
+
+__all__ = ["__version__"]

rosetta_cli/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for running rosetta-cli as a module."""
+
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

rosetta_cli/cli.py

@@ -0,0 +1,379 @@
+"""Rosetta CLI entry point."""
+
+
+import argparse
+import sys
+from collections.abc import Callable
+from typing import TypeAlias
+
+from .commands.base_command import BaseCommand
+from .commands.cleanup_command import CleanupCommand
+from .commands.list_command import ListCommand
+from .commands.parse_command import ParseCommand
+from .commands.publish_command import PublishCommand
+from .commands.verify_command import VerifyCommand
+from .ims_config import IMSConfig
+from .ragflow_client import RAGFlowClient
+from .typing_utils import CommandArgs
+
+CommandClass: TypeAlias = Callable[[RAGFlowClient, IMSConfig], BaseCommand]
+
+
+# Command registry mapping command names to their classes
+COMMAND_REGISTRY: dict[str, CommandClass] = {
+    'publish': PublishCommand,
+    'verify': VerifyCommand,
+    'list-dataset': ListCommand,
+    'cleanup-dataset': CleanupCommand,
+    'parse': ParseCommand,
+}
+
+
+def execute_command(command_name: str, args: CommandArgs, client: RAGFlowClient, config: IMSConfig) -> int:
+    """
+    Execute a command by name using the command registry.
+    
+    Args:
+        command_name: Name of the command to execute
+        args: Parsed command-line arguments
+        client: RAGFlow client instance
+        config: RAGFlow configuration
+    
+    Returns:
+        Exit code (0 for success, 1 for failure)
+    """
+    command_class = COMMAND_REGISTRY.get(command_name)
+    if not command_class:
+        print(f"Unknown command: {command_name}")
+        return 1
+    
+    command = command_class(client, config)
+    return command.execute(args)
+
+
+def main() -> int:
+    """Main CLI entry point."""
+    parser = argparse.ArgumentParser(
+        description="Rosetta CLI - Publish knowledge base content to RAGFlow and manage datasets\n"
+                    "All commands include performance timing measurements.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Publish knowledge base content from a folder (with timing)
+  rosetta-cli publish ../instructions
+
+  # Publish with dry-run (no actual upload)
+  rosetta-cli publish ../business --dry-run
+
+  # Force republish (ignore change detection)
+  rosetta-cli publish ../instructions --force
+
+  # List documents in dataset
+  rosetta-cli list-dataset
+
+  # List documents in specific dataset
+  rosetta-cli list-dataset --dataset aia-r1
+
+  # Cleanup (delete all documents from) a dataset
+  rosetta-cli cleanup-dataset --dataset aia-r1
+
+  # Preview cleanup with dry-run (shows what would be deleted)
+  rosetta-cli cleanup-dataset --dataset aia-r1 --dry-run
+
+  # Cleanup documents with specific prefix
+  rosetta-cli cleanup-dataset --dataset aia-r1 --prefix "aqa-phase"
+
+  # Cleanup documents with specific tags (space-separated)
+  rosetta-cli cleanup-dataset --dataset aia-r1 --tags "r1 agents"
+
+  # Cleanup documents with specific tags (comma-separated)
+  rosetta-cli cleanup-dataset --dataset aia-r1 --tags "r1,agents"
+
+  # Preview cleanup with prefix filter
+  rosetta-cli cleanup-dataset --dataset aia-r1 --prefix "aqa-phase" --dry-run
+
+  # Preview cleanup with tags filter
+  rosetta-cli cleanup-dataset --dataset aia-r1 --tags "r1 agents" --dry-run
+
+  # Cleanup with force (skip confirmation)
+  rosetta-cli cleanup-dataset --dataset aia-r1 --force
+  
+  # Cleanup with prefix and force
+  rosetta-cli cleanup-dataset --dataset aia-r1 --prefix "aqa-phase" --force
+
+  # Cleanup with tags and force
+  rosetta-cli cleanup-dataset --dataset aia-r1 --tags "r1,agents" --force
+
+  # Parse documents in dataset (retry failed/unparsed)
+  rosetta-cli parse --dataset aia-r1
+
+  # Force re-parse all documents (e.g., after changing parser config)
+  rosetta-cli parse --dataset aia-r1 --force
+
+  # Preview which documents would be parsed
+  rosetta-cli parse --dataset aia-r1 --dry-run
+
+  # Verify connection
+  rosetta-cli verify
+
+  # Use different environment
+  rosetta-cli publish ../instructions --env production
+
+Performance Notes:
+  - All commands show execution time (⏱️ Total time: X.XXs)
+  - Publishing ~10-15s per file (embedding generation)
+  - Change detection skips unchanged files (77% faster)
+  - API key verification timing shown when applicable
+
+Tag-in-Title Format:
+  - Documents are published with tags in title: [tag1][tag2] filename.ext
+  - Example: [instructions][agents][r1] agents.md
+
+Frontmatter Metadata (publish flow):
+  - Supported keys: tags, sort_order
+  - tags can be list or comma-separated string
+  - tags are merged with path-based tags (case-insensitive dedupe)
+  - sort_order is persisted to metadata and affects MCP bundling order
+  - original_path/resource_path are normalized from instructions-relative path when applicable
+"""
+    )
+    
+    # Global arguments
+    parser.add_argument(
+        '--env',
+        type=str,
+        default=None,
+        help='Environment (local, dev, test, production)'
+    )
+    parser.add_argument(
+        '--env-file',
+        type=str,
+        default=None,
+        help='Explicit path to a .env file'
+    )
+    
+    # Subcommands
+    subparsers = parser.add_subparsers(dest='command', help='Command to execute')
+    
+    # Publish command
+    publish_parser = subparsers.add_parser(
+        'publish',
+        help='Publish knowledge base content to RAGFlow'
+    )
+    publish_parser.add_argument(
+        'path',
+        type=str,
+        help='Path to content file or folder (e.g., instructions/, business/)'
+    )
+    publish_parser.add_argument(
+        '--dry-run',
+        action='store_true',
+        help='Simulate publishing without actual upload'
+    )
+    publish_parser.add_argument(
+        '--force',
+        action='store_true',
+        help='Force republish all files, ignoring change detection'
+    )
+    publish_parser.add_argument(
+        '--no-parse',
+        action='store_true',
+        help='Skip parsing documents after upload (for debugging)'
+    )
+    publish_parser.add_argument(
+        '--parse-timeout',
+        type=int,
+        default=300,
+        help='Timeout for parsing in seconds (default: 300)'
+    )
+    publish_parser.add_argument(
+        '--env',
+        type=str,
+        default=None,
+        help='Environment (local, dev, test, production)'
+    )
+    publish_parser.add_argument(
+        '--env-file',
+        type=str,
+        default=None,
+        help='Explicit path to a .env file'
+    )
+    
+    # Verify command
+    verify_parser = subparsers.add_parser(
+        'verify',
+        help='Verify RAGFlow connection and API key'
+    )
+    verify_parser.add_argument(
+        '--env',
+        type=str,
+        default=None,
+        help='Environment (local, dev, test, production)'
+    )
+    verify_parser.add_argument(
+        '--env-file',
+        type=str,
+        default=None,
+        help='Explicit path to a .env file'
+    )
+    
+    # List dataset command
+    list_parser = subparsers.add_parser(
+        'list-dataset',
+        help='List documents in a dataset'
+    )
+    list_parser.add_argument(
+        '--dataset',
+        type=str,
+        default=None,
+        help='Dataset name (defaults to configured dataset)'
+    )
+    list_parser.add_argument(
+        '--env',
+        type=str,
+        default=None,
+        help='Environment (local, dev, test, production)'
+    )
+    list_parser.add_argument(
+        '--env-file',
+        type=str,
+        default=None,
+        help='Explicit path to a .env file'
+    )
+    
+    # Cleanup dataset command
+    cleanup_parser = subparsers.add_parser(
+        'cleanup-dataset',
+        help='Delete all documents from a dataset'
+    )
+    cleanup_parser.add_argument(
+        '--dataset',
+        type=str,
+        default=None,
+        help='Dataset name (defaults to configured dataset)'
+    )
+    cleanup_parser.add_argument(
+        '--prefix',
+        type=str,
+        default=None,
+        help='Only delete documents with titles starting with this prefix (e.g., "aqa-phase")'
+    )
+    cleanup_parser.add_argument(
+        '--tags',
+        type=str,
+        default=None,
+        help='Only delete documents with these tags (space or comma separated, e.g., "r1 agents" or "r1,agents")'
+    )
+    cleanup_parser.add_argument(
+        '--dry-run',
+        action='store_true',
+        help='Show what would be deleted without actually deleting'
+    )
+    cleanup_parser.add_argument(
+        '--force',
+        action='store_true',
+        help='Skip confirmation prompt'
+    )
+    cleanup_parser.add_argument(
+        '--env',
+        type=str,
+        default=None,
+        help='Environment (local, dev, test, production)'
+    )
+    cleanup_parser.add_argument(
+        '--env-file',
+        type=str,
+        default=None,
+        help='Explicit path to a .env file'
+    )
+    
+    # Parse command
+    parse_parser = subparsers.add_parser(
+        'parse',
+        help='Trigger parsing for documents in a dataset'
+    )
+    parse_parser.add_argument(
+        '--dataset',
+        type=str,
+        default=None,
+        help='Dataset name (defaults to configured dataset)'
+    )
+    parse_parser.add_argument(
+        '--force',
+        action='store_true',
+        help='Force re-parse ALL documents, even if already parsed'
+    )
+    parse_parser.add_argument(
+        '--dry-run',
+        action='store_true',
+        help='Show which documents would be parsed without actually parsing'
+    )
+    parse_parser.add_argument(
+        '--yes',
+        action='store_true',
+        help='Skip confirmation prompt'
+    )
+    parse_parser.add_argument(
+        '--parse-timeout',
+        type=int,
+        default=300,
+        help='Timeout for parsing in seconds (default: 300)'
+    )
+    parse_parser.add_argument(
+        '--env',
+        type=str,
+        default=None,
+        help='Environment (local, dev, test, production)'
+    )
+    parse_parser.add_argument(
+        '--env-file',
+        type=str,
+        default=None,
+        help='Explicit path to a .env file'
+    )
+    
+    # Parse arguments
+    args = parser.parse_args()
+    
+    # Check if command was provided
+    if not args.command:
+        parser.print_help()
+        return 1
+    
+    try:
+        # Load configuration
+        config = IMSConfig.from_env(env_file=args.env_file, environment=args.env)
+        
+        # Validate configuration
+        config.validate()
+        
+        # Initialize RAGFlow client
+        client = RAGFlowClient(
+            api_key=config.api_key,
+            base_url=config.base_url,
+            embedding_model=config.embedding_model,
+            chunk_method=config.chunk_method,
+            parser_config=config.parser_config,
+            page_size=config.page_size
+        )
+        
+        return execute_command(args.command, args, client, config)
+            
+    except ValueError as e:
+        print(f"✗ Configuration error: {e}")
+        print("\nPlease ensure you have:")
+        print("1. Created a .env file (copy from env.template)")
+        print("2. Set RAGFLOW_BASE_URL and RAGFLOW_API_KEY")
+        return 1
+    except KeyboardInterrupt:
+        print("\n\nOperation cancelled.")
+        return 1
+    except Exception as e:
+        print(f"✗ Unexpected error: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main())

rosetta_cli/commands/__init__.py

@@ -0,0 +1,5 @@
+"""IMS Commands Package - Command pattern implementations for IMS CLI"""
+
+from .base_command import BaseCommand
+
+__all__ = ['BaseCommand']

rosetta_cli/commands/base_command.py

@@ -0,0 +1,82 @@
+"""
+Base Command Abstract Class
+
+Defines the interface and common functionality for all IMS CLI commands.
+"""
+
+import time
+from abc import ABC, abstractmethod
+
+from ..ims_config import IMSConfig
+from ..ragflow_client import RAGFlowClient
+from ..typing_utils import CommandArgs
+
+
+class BaseCommand(ABC):
+    """
+    Abstract base class for all IMS CLI commands.
+    
+    Provides common functionality for authentication, timing, and error handling.
+    Each command must implement the execute() method.
+    """
+    
+    def __init__(self, client: RAGFlowClient, config: IMSConfig):
+        """
+        Initialize command with RAGFlow client and configuration.
+        
+        Args:
+            client: RAGFlow client instance
+            config: IMS configuration
+        """
+        self.client = client
+        self.config = config
+        self._start_time: float | None = None
+    
+    @abstractmethod
+    def execute(self, args: CommandArgs) -> int:
+        """
+        Execute the command with given arguments.
+        
+        Args:
+            args: Parsed command-line arguments (argparse.Namespace)
+        
+        Returns:
+            Exit code (0 for success, non-zero for failure)
+        """
+        raise NotImplementedError
+    
+    def _start_timing(self) -> None:
+        """Start timing measurement for command execution."""
+        self._start_time = time.time()
+    
+    def _get_elapsed_time(self) -> float:
+        """
+        Get elapsed time since timing started.
+        
+        Returns:
+            Elapsed time in seconds
+        """
+        if self._start_time is None:
+            return 0.0
+        return time.time() - self._start_time
+    
+    def _print_timing(self, label: str = "Total time") -> None:
+        """
+        Print timing information.
+        
+        Args:
+            label: Label for the timing output
+        """
+        elapsed = self._get_elapsed_time()
+        print(f"⏱️  {label}: {elapsed:.2f}s")
+    
+    def _print_header(self, title: str) -> None:
+        """
+        Print command header with configuration info.
+        
+        Args:
+            title: Command title/description
+        """
+        print(title)
+        print(f"Environment: {self.config.environment}")
+        print(f"RAGFlow Instance: {self.config.base_url}")

rosetta_cli/commands/cleanup_command.py

@@ -0,0 +1,214 @@
+"""
+Cleanup Command - Delete documents from a dataset
+"""
+
+from ..services.dataset_service import DatasetService
+from ..services.document_service import DocumentService
+
+from .base_command import BaseCommand
+from ..typing_utils import CommandArgs, DatasetLike, DocumentLike
+
+
+class CleanupCommand(BaseCommand):
+    """
+    Delete documents from a RAGFlow dataset.
+    
+    CRITICAL: Implements safety measures (dry-run, confirmation) to prevent accidental data loss.
+    """
+    
+    def execute(self, args: CommandArgs) -> int:
+        """Execute cleanup-dataset command."""
+        self._start_timing()
+        
+        # Resolve dataset name
+        dataset_service = DatasetService(self.client, self.config)
+        dataset_name, auto_detected = dataset_service.resolve_dataset_name(args.dataset)
+        
+        if not dataset_name:
+            return 1
+        
+        # Print header
+        print(f"Cleaning up Dataset: {dataset_name}")
+        print(f"Environment: {self.config.environment}")
+        print(f"RAGFlow Instance: {self.config.base_url}\n")
+        
+        # Verify authentication
+        from services.auth_service import AuthService
+        AuthService.verify_or_exit(self.client, self.config)
+        
+        try:
+            # Get dataset
+            dataset = self.client.get_dataset(name=dataset_name)
+            
+            if not dataset:
+                print(f"✗ Dataset '{dataset_name}' not found")
+                dataset_service.display_available_datasets()
+                return 1
+            
+            # Filter documents
+            filtered_documents = self._get_filtered_documents(
+                dataset, dataset_service, args
+            )
+            
+            if not filtered_documents:
+                print(f"✓ No documents found")
+                return 0
+            
+            # Show documents
+            self._display_documents(filtered_documents)
+            
+            # Dry-run mode
+            if args.dry_run:
+                return self._handle_dry_run(filtered_documents)
+            
+            # Confirm deletion
+            if not self._confirm_deletion(filtered_documents, dataset_name, args):
+                print("\n✗ Cleanup cancelled")
+                return 1
+            
+            # Delete documents
+            deleted, failed = self._delete_documents(dataset, filtered_documents)
+            
+            # Print summary
+            self._print_summary(deleted, failed)
+            
+            return 0 if failed == 0 else 1
+            
+        except Exception as e:
+            print(f"\n✗ Error cleaning up dataset: {e}")
+            self._print_timing()
+            import traceback
+            traceback.print_exc()
+            return 1
+    
+    def _get_filtered_documents(
+        self,
+        dataset: DatasetLike,
+        dataset_service: DatasetService,
+        args: CommandArgs,
+    ) -> list[DocumentLike]:
+        """Get documents to delete based on filters."""
+        document_service = DocumentService(self.client)
+        
+        if args.tags:
+            # Filter by tags (metadata condition)
+            tags_list = self._parse_tags(args.tags)
+            filtered_documents = document_service.filter_documents_by_tags(
+                dataset, tags_list
+            )
+            print(f"\nFiltered {len(filtered_documents)} document(s) with tags: {', '.join(tags_list)}\n")
+        elif args.prefix:
+            # Filter by prefix
+            filtered_documents = document_service.filter_documents_by_prefix(
+                dataset, args.prefix
+            )
+            print(f"\nFiltered {len(filtered_documents)} document(s) matching prefix '{args.prefix}'\n")
+        else:
+            # No filter - fetch all
+            filtered_documents = dataset.list_documents(page_size=self.config.page_size)
+            if filtered_documents:
+                print(f"\nFound {len(filtered_documents)} document(s) to delete\n")
+        
+        return filtered_documents
+    
+    def _parse_tags(self, tags_arg: str) -> list[str]:
+        """Parse tags from comma or space separated string."""
+        # Support both comma and space separated tags
+        if ',' in tags_arg:
+            tags = [tag.strip() for tag in tags_arg.split(',')]
+        else:
+            tags = tags_arg.split()
+        
+        # Filter out empty strings
+        return [tag for tag in tags if tag]
+    
+    def _display_documents(self, documents: list[DocumentLike]) -> None:
+        """Display documents that will be deleted."""
+        print("Documents to delete:")
+        print("="*80)
+        for document in documents:
+            doc_name = document.name if hasattr(document, 'name') else 'Untitled'
+            print(f"  • {doc_name}")
+        print("="*80)
+        print()
+    
+    def _handle_dry_run(self, documents: list[DocumentLike]) -> int:
+        """Handle dry-run mode."""
+        print("🔍 DRY-RUN MODE - No documents will be deleted\n")
+        print(f"Summary: {len(documents)} document(s) would be deleted")
+        print("="*80)
+        self._print_timing()
+        return 0
+    
+    def _confirm_deletion(
+        self,
+        documents: list[DocumentLike],
+        dataset_name: str,
+        args: CommandArgs,
+    ) -> bool:
+        """Confirm deletion with user (unless force flag set)."""
+        if args.force:
+            return True
+        
+        # Build filter message
+        filter_msg = ""
+        if args.tags:
+            tags_list = self._parse_tags(args.tags)
+            filter_msg = f" with tags ({', '.join(tags_list)})"
+        elif args.prefix:
+            filter_msg = f" with prefix '{args.prefix}'"
+        
+        response = input(
+            f"⚠️  Delete {len(documents)} documents{filter_msg} from '{dataset_name}'? (yes/no): "
+        )
+        return response.lower() in ['yes', 'y']
+    
+    def _delete_documents(
+        self,
+        dataset: DatasetLike,
+        documents: list[DocumentLike],
+    ) -> tuple[int, int]:
+        """Delete documents and return counts."""
+        deleted_count = 0
+        failed_count = 0
+        
+        # Collect document IDs
+        doc_ids_to_delete = []
+        doc_id_to_name = {}
+        
+        for document in documents:
+            doc_id = document.id if hasattr(document, 'id') else None
+            doc_name = document.name if hasattr(document, 'name') else 'Untitled'
+            
+            if not doc_id:
+                print(f"✗ Cannot delete {doc_name}: No document ID")
+                failed_count += 1
+                continue
+            
+            doc_ids_to_delete.append(doc_id)
+            doc_id_to_name[doc_id] = doc_name
+        
+        # Delete batch
+        if doc_ids_to_delete:
+            try:
+                dataset.delete_documents(ids=doc_ids_to_delete)
+                deleted_count = len(doc_ids_to_delete)
+                
+                for doc_id in doc_ids_to_delete:
+                    doc_name = doc_id_to_name.get(doc_id, 'Unknown')
+                    print(f"✓ Deleted: {doc_name}")
+                    
+            except Exception as e:
+                print(f"✗ Failed to delete documents: {e}")
+                failed_count = len(doc_ids_to_delete)
+        
+        return deleted_count, failed_count
+    
+    def _print_summary(self, deleted: int, failed: int) -> None:
+        """Print cleanup summary."""
+        print("\n" + "="*80)
+        print(f"Cleanup Summary:")
+        print(f"  ✓ Deleted: {deleted}")
+        print(f"  ✗ Failed: {failed}")
+        print("="*80)
+        self._print_timing()