aiecs 1.0.0__py3-none-any.whl

aiecs/scripts/dependency_fixer.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python3
+"""
+Dependency fixer for AIECS tools.
+
+This script automatically installs missing dependencies based on the
+dependency checker results.
+"""
+
+import os
+import sys
+import subprocess
+import platform
+import logging
+from typing import Dict, List, Tuple, Optional
+from pathlib import Path
+
+
+class DependencyFixer:
+    """Automatic dependency fixer for AIECS."""
+    
+    def __init__(self, interactive: bool = True):
+        self.logger = self._setup_logging()
+        self.system = platform.system().lower()
+        self.interactive = interactive
+        self.fixes_applied = []
+        self.fixes_failed = []
+        
+    def _setup_logging(self) -> logging.Logger:
+        """Setup logging configuration."""
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(levelname)s - %(message)s',
+            handlers=[
+                logging.StreamHandler(sys.stdout),
+                logging.FileHandler('dependency_fix.log')
+            ]
+        )
+        return logging.getLogger(__name__)
+    
+    def _run_command(self, cmd: List[str], description: str) -> bool:
+        """Run a command and return success status."""
+        try:
+            self.logger.info(f"Running: {description}")
+            self.logger.info(f"Command: {' '.join(cmd)}")
+            
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                check=True,
+                timeout=300  # 5 minutes timeout
+            )
+            
+            self.logger.info(f"Success: {description}")
+            if result.stdout:
+                self.logger.info(f"Output: {result.stdout}")
+            
+            return True
+            
+        except subprocess.CalledProcessError as e:
+            self.logger.error(f"Failed: {description}")
+            self.logger.error(f"Error: {e.stderr}")
+            return False
+        except subprocess.TimeoutExpired:
+            self.logger.error(f"Timeout: {description}")
+            return False
+        except Exception as e:
+            self.logger.error(f"Error: {description} - {e}")
+            return False
+    
+    def _ask_confirmation(self, message: str) -> bool:
+        """Ask for user confirmation if in interactive mode."""
+        if not self.interactive:
+            return True
+        
+        while True:
+            response = input(f"{message} (y/n): ").lower().strip()
+            if response in ['y', 'yes']:
+                return True
+            elif response in ['n', 'no']:
+                return False
+            else:
+                print("Please enter 'y' or 'n'")
+    
+    def fix_system_dependencies(self, missing_deps: List[str]) -> bool:
+        """Fix missing system dependencies."""
+        if not missing_deps:
+            return True
+        
+        self.logger.info("Fixing system dependencies...")
+        
+        # Group dependencies by package manager
+        apt_packages = []
+        brew_packages = []
+        pip_packages = []
+        
+        for dep in missing_deps:
+            if dep == "java":
+                if self.system == "linux":
+                    apt_packages.append("openjdk-11-jdk")
+                elif self.system == "darwin":
+                    brew_packages.append("openjdk@11")
+            elif dep == "tesseract":
+                if self.system == "linux":
+                    apt_packages.extend(["tesseract-ocr", "tesseract-ocr-eng"])
+                elif self.system == "darwin":
+                    brew_packages.append("tesseract")
+            elif dep == "tesseract_lang_packs":
+                if self.system == "linux":
+                    apt_packages.extend([
+                        "tesseract-ocr-chi-sim", "tesseract-ocr-chi-tra",
+                        "tesseract-ocr-fra", "tesseract-ocr-deu",
+                        "tesseract-ocr-jpn", "tesseract-ocr-kor",
+                        "tesseract-ocr-rus", "tesseract-ocr-spa"
+                    ])
+            elif dep == "pillow_system_deps":
+                if self.system == "linux":
+                    apt_packages.extend([
+                        "libjpeg-dev", "zlib1g-dev", "libpng-dev",
+                        "libtiff-dev", "libwebp-dev", "libopenjp2-7-dev"
+                    ])
+                elif self.system == "darwin":
+                    brew_packages.extend(["libjpeg", "zlib", "libpng", "libtiff", "webp", "openjpeg"])
+            elif dep == "pyreadstat_deps":
+                if self.system == "linux":
+                    apt_packages.append("libreadstat-dev")
+                elif self.system == "darwin":
+                    brew_packages.append("readstat")
+            elif dep == "weasyprint_deps":
+                if self.system == "linux":
+                    apt_packages.extend([
+                        "libcairo2-dev", "libpango1.0-dev", "libgdk-pixbuf2.0-dev",
+                        "libffi-dev", "shared-mime-info"
+                    ])
+                elif self.system == "darwin":
+                    brew_packages.extend(["cairo", "pango", "gdk-pixbuf", "libffi"])
+            elif dep == "matplotlib_deps":
+                if self.system == "linux":
+                    apt_packages.extend([
+                        "libfreetype6-dev", "libpng-dev", "libjpeg-dev",
+                        "libtiff-dev", "libwebp-dev"
+                    ])
+                elif self.system == "darwin":
+                    brew_packages.extend(["freetype", "libpng", "libjpeg", "libtiff", "webp"])
+        
+        # Install apt packages
+        if apt_packages and self.system == "linux":
+            if self._ask_confirmation(f"Install system packages: {', '.join(apt_packages)}?"):
+                cmd = ["sudo", "apt-get", "update"]
+                if self._run_command(cmd, "Update package list"):
+                    cmd = ["sudo", "apt-get", "install", "-y"] + apt_packages
+                    if self._run_command(cmd, f"Install packages: {', '.join(apt_packages)}"):
+                        self.fixes_applied.append(f"System packages: {', '.join(apt_packages)}")
+                    else:
+                        self.fixes_failed.append(f"System packages: {', '.join(apt_packages)}")
+                        return False
+        
+        # Install brew packages
+        if brew_packages and self.system == "darwin":
+            if self._ask_confirmation(f"Install Homebrew packages: {', '.join(brew_packages)}?"):
+                for package in brew_packages:
+                    cmd = ["brew", "install", package]
+                    if self._run_command(cmd, f"Install {package}"):
+                        self.fixes_applied.append(f"Homebrew package: {package}")
+                    else:
+                        self.fixes_failed.append(f"Homebrew package: {package}")
+                        return False
+        
+        return True
+    
+    def fix_python_dependencies(self, missing_packages: List[str]) -> bool:
+        """Fix missing Python packages."""
+        if not missing_packages:
+            return True
+        
+        self.logger.info("Fixing Python dependencies...")
+        
+        if self._ask_confirmation(f"Install Python packages: {', '.join(missing_packages)}?"):
+            cmd = [sys.executable, "-m", "pip", "install"] + missing_packages
+            if self._run_command(cmd, f"Install Python packages: {', '.join(missing_packages)}"):
+                self.fixes_applied.append(f"Python packages: {', '.join(missing_packages)}")
+                return True
+            else:
+                self.fixes_failed.append(f"Python packages: {', '.join(missing_packages)}")
+                return False
+        
+        return True
+    
+    def fix_model_dependencies(self, missing_models: List[str]) -> bool:
+        """Fix missing model dependencies."""
+        if not missing_models:
+            return True
+        
+        self.logger.info("Fixing model dependencies...")
+        
+        # spaCy models
+        spacy_models = [m for m in missing_models if m.startswith("spacy_")]
+        if spacy_models:
+            if self._ask_confirmation(f"Download spaCy models: {', '.join(spacy_models)}?"):
+                for model in spacy_models:
+                    model_name = model.replace("spacy_", "")
+                    cmd = [sys.executable, "-m", "spacy", "download", model_name]
+                    if self._run_command(cmd, f"Download spaCy model: {model_name}"):
+                        self.fixes_applied.append(f"spaCy model: {model_name}")
+                    else:
+                        self.fixes_failed.append(f"spaCy model: {model_name}")
+        
+        # NLTK data
+        nltk_data = [m for m in missing_models if m.startswith("nltk_")]
+        if nltk_data:
+            if self._ask_confirmation(f"Download NLTK data: {', '.join(nltk_data)}?"):
+                for data in nltk_data:
+                    data_name = data.replace("nltk_", "")
+                    cmd = [sys.executable, "-c", f"import nltk; nltk.download('{data_name}')"]
+                    if self._run_command(cmd, f"Download NLTK data: {data_name}"):
+                        self.fixes_applied.append(f"NLTK data: {data_name}")
+                    else:
+                        self.fixes_failed.append(f"NLTK data: {data_name}")
+        
+        # Playwright browsers
+        if "playwright_browsers" in missing_models:
+            if self._ask_confirmation("Install Playwright browsers?"):
+                cmd = [sys.executable, "-m", "playwright", "install"]
+                if self._run_command(cmd, "Install Playwright browsers"):
+                    self.fixes_applied.append("Playwright browsers")
+                else:
+                    self.fixes_failed.append("Playwright browsers")
+        
+        return True
+    
+    def fix_dependencies_from_checker(self, checker_results: Dict) -> bool:
+        """Fix dependencies based on checker results."""
+        self.logger.info("Starting dependency fixing process...")
+        
+        # Extract missing dependencies
+        missing_system = []
+        missing_python = []
+        missing_models = []
+        
+        for tool_name, tool_deps in checker_results.items():
+            if isinstance(tool_deps, dict):
+                # Handle tool-specific dependencies
+                for dep_type, deps in tool_deps.items():
+                    if dep_type == "system_deps":
+                        for dep in deps:
+                            if dep.status.value == "missing":
+                                missing_system.append(dep.name.lower().replace(" ", "_"))
+                    elif dep_type == "python_deps":
+                        for dep in deps:
+                            if dep.status.value == "missing":
+                                missing_python.append(dep.name)
+                    elif dep_type == "model_deps":
+                        for dep in deps:
+                            if dep.status.value == "missing":
+                                missing_models.append(f"{dep.name.lower().replace(' ', '_')}")
+        
+        # Apply fixes
+        success = True
+        
+        if missing_system:
+            if not self.fix_system_dependencies(missing_system):
+                success = False
+        
+        if missing_python:
+            if not self.fix_python_dependencies(missing_python):
+                success = False
+        
+        if missing_models:
+            if not self.fix_model_dependencies(missing_models):
+                success = False
+        
+        return success
+    
+    def generate_fix_report(self) -> str:
+        """Generate a report of fixes applied."""
+        report = []
+        report.append("=" * 60)
+        report.append("AIECS DEPENDENCY FIX REPORT")
+        report.append("=" * 60)
+        
+        if self.fixes_applied:
+            report.append("\n✅ Successfully Applied Fixes:")
+            for fix in self.fixes_applied:
+                report.append(f"  • {fix}")
+        
+        if self.fixes_failed:
+            report.append("\n❌ Failed Fixes:")
+            for fix in self.fixes_failed:
+                report.append(f"  • {fix}")
+        
+        if not self.fixes_applied and not self.fixes_failed:
+            report.append("\nℹ️  No fixes were applied.")
+        
+        report.append(f"\nTotal fixes applied: {len(self.fixes_applied)}")
+        report.append(f"Total fixes failed: {len(self.fixes_failed)}")
+        
+        if self.fixes_failed:
+            report.append("\n⚠️  Some fixes failed. Please check the logs and try manual installation.")
+        else:
+            report.append("\n🎉 All fixes applied successfully!")
+        
+        return "\n".join(report)
+
+
+def main():
+    """Main function."""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Fix AIECS dependencies")
+    parser.add_argument("--non-interactive", action="store_true", 
+                       help="Run in non-interactive mode (auto-approve all fixes)")
+    parser.add_argument("--check-only", action="store_true",
+                       help="Only check dependencies, don't fix them")
+    
+    args = parser.parse_args()
+    
+    # Import and run dependency checker first
+    try:
+        from aiecs.scripts.dependency_checker import DependencyChecker
+        checker = DependencyChecker()
+        tools = checker.check_all_dependencies()
+        
+        if args.check_only:
+            report = checker.generate_report(tools)
+            print(report)
+            return 0
+        
+        # Run fixer
+        fixer = DependencyFixer(interactive=not args.non_interactive)
+        success = fixer.fix_dependencies_from_checker(tools)
+        
+        # Generate and display report
+        report = fixer.generate_fix_report()
+        print(report)
+        
+        return 0 if success else 1
+        
+    except Exception as e:
+        print(f"Error: {e}")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
+
+
+
+

aiecs/scripts/download_nlp_data.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python3
+"""
+Automated script to download required NLP data for AIECS ClassifierTool.
+
+This script downloads:
+1. NLTK stopwords data package for keyword extraction
+2. spaCy English model (en_core_web_sm) for text processing
+3. spaCy Chinese model (zh_core_web_sm) for Chinese text processing
+"""
+
+import os
+import sys
+import subprocess
+import logging
+from pathlib import Path
+from typing import List, Tuple, Optional
+
+
+def setup_logging():
+    """Setup logging configuration."""
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        handlers=[
+            logging.StreamHandler(sys.stdout),
+            logging.FileHandler('nlp_data_download.log')
+        ]
+    )
+    return logging.getLogger(__name__)
+
+
+def run_command(cmd: List[str], logger: logging.Logger) -> Tuple[bool, str]:
+    """
+    Run a shell command and return success status and output.
+    
+    Args:
+        cmd: List of command arguments
+        logger: Logger instance
+    
+    Returns:
+        Tuple of (success, output)
+    """
+    try:
+        logger.info(f"Running command: {' '.join(cmd)}")
+        result = subprocess.run(
+            cmd, 
+            capture_output=True, 
+            text=True, 
+            check=True
+        )
+        logger.info(f"Command succeeded: {result.stdout}")
+        return True, result.stdout
+    except subprocess.CalledProcessError as e:
+        error_msg = f"Command failed with exit code {e.returncode}: {e.stderr}"
+        logger.error(error_msg)
+        return False, error_msg
+    except FileNotFoundError:
+        error_msg = f"Command not found: {cmd[0]}"
+        logger.error(error_msg)
+        return False, error_msg
+
+
+def check_python_package(package_name: str, logger: logging.Logger) -> bool:
+    """
+    Check if a Python package is installed.
+    
+    Args:
+        package_name: Name of the package to check
+        logger: Logger instance
+        
+    Returns:
+        True if package is installed, False otherwise
+    """
+    try:
+        __import__(package_name)
+        logger.info(f"Package {package_name} is already installed")
+        return True
+    except ImportError:
+        logger.warning(f"Package {package_name} is not installed")
+        return False
+
+
+def download_nltk_data(logger: logging.Logger) -> bool:
+    """
+    Download required NLTK data packages.
+    
+    Args:
+        logger: Logger instance
+    
+    Returns:
+        True if successful, False otherwise
+    """
+    logger.info("Starting NLTK data download...")
+    
+    if not check_python_package('nltk', logger):
+        logger.error("NLTK is not installed. Please install it first with: pip install nltk")
+        return False
+    
+    try:
+        import nltk
+        
+        # Download required NLTK data
+        packages_to_download = [
+            'stopwords',
+            'punkt',
+            'wordnet',
+            'averaged_perceptron_tagger'
+        ]
+        
+        for package in packages_to_download:
+            try:
+                logger.info(f"Downloading NLTK package: {package}")
+                nltk.download(package, quiet=True)
+                logger.info(f"Successfully downloaded NLTK package: {package}")
+            except Exception as e:
+                logger.error(f"Failed to download NLTK package {package}: {e}")
+                return False
+                
+        logger.info("All NLTK data packages downloaded successfully")
+        return True
+        
+    except Exception as e:
+        logger.error(f"Error downloading NLTK data: {e}")
+        return False
+
+
+def download_spacy_model(model_name: str, logger: logging.Logger) -> bool:
+    """
+    Download a spaCy model.
+    
+    Args:
+        model_name: Name of the spaCy model to download
+        logger: Logger instance
+    
+    Returns:
+        True if successful, False otherwise
+    """
+    logger.info(f"Starting spaCy model download: {model_name}")
+    
+    if not check_python_package('spacy', logger):
+        logger.error("spaCy is not installed. Please install it first with: pip install spacy")
+        return False
+    
+    # Check if model is already installed
+    try:
+        import spacy
+        spacy.load(model_name)
+        logger.info(f"spaCy model {model_name} is already installed")
+        return True
+    except OSError:
+        # Model not installed, proceed with download
+        pass
+    except Exception as e:
+        logger.error(f"Error checking spaCy model {model_name}: {e}")
+        return False
+    
+    # Download the model
+    cmd = [sys.executable, "-m", "spacy", "download", model_name]
+    success, output = run_command(cmd, logger)
+    
+    if success:
+        logger.info(f"Successfully downloaded spaCy model: {model_name}")
+        
+        # Verify the model can be loaded
+        try:
+            import spacy
+            spacy.load(model_name)
+            logger.info(f"Verified spaCy model {model_name} can be loaded")
+            return True
+        except Exception as e:
+            logger.error(f"Downloaded model {model_name} cannot be loaded: {e}")
+            return False
+    else:
+        logger.error(f"Failed to download spaCy model {model_name}: {output}")
+        return False
+    
+
+def download_spacy_pkuseg_model(logger: logging.Logger) -> bool:
+    """
+    Download and install spaCy PKUSeg model for Chinese text segmentation.
+
+    Args:
+        logger: Logger instance
+
+    Returns:
+        True if successful, False otherwise
+    """
+    logger.info("Starting spaCy PKUSeg model installation...")
+
+    if not check_python_package('spacy', logger):
+        logger.error("spaCy is not installed. Please install it first with: pip install spacy")
+        return False
+
+    # Check if spacy_pkuseg is already installed
+    if check_python_package('spacy_pkuseg', logger):
+        logger.info("spacy_pkuseg is already installed")
+        return True
+
+    # Install spacy_pkuseg package
+    cmd = [sys.executable, "-m", "pip", "install", "spacy_pkuseg"]
+    success, output = run_command(cmd, logger)
+
+    if success:
+        logger.info("Successfully installed spacy_pkuseg")
+
+        # Verify the package can be imported
+        try:
+            import spacy_pkuseg
+            logger.info("Verified spacy_pkuseg can be imported")
+
+            # Test basic functionality
+            seg = spacy_pkuseg.pkuseg()
+            test_result = seg.cut("这是一个测试句子")
+            logger.info(f"spacy_pkuseg test successful: {list(test_result)}")
+            return True
+        except Exception as e:
+            logger.error(f"Installed spacy_pkuseg cannot be used: {e}")
+            return False
+    else:
+        logger.error(f"Failed to install spacy_pkuseg: {output}")
+        return False
+
+
+def download_rake_nltk_data(logger: logging.Logger) -> bool:
+    """
+    Ensure RAKE-NLTK has required data.
+    
+    Args:
+        logger: Logger instance
+    
+    Returns:
+        True if successful, False otherwise
+    """
+    logger.info("Checking RAKE-NLTK data...")
+    
+    if not check_python_package('rake_nltk', logger):
+        logger.warning("RAKE-NLTK is not installed. This is optional for English keyword extraction.")
+        return True  # Not critical, return True
+    
+    try:
+        from rake_nltk import Rake
+        # Test RAKE functionality
+        rake = Rake()
+        rake.extract_keywords_from_text("This is a test sentence for RAKE.")
+        keywords = rake.get_ranked_phrases()
+        logger.info("RAKE-NLTK is working correctly")
+        return True
+    except Exception as e:
+        logger.warning(f"RAKE-NLTK test failed: {e}. This is not critical.")
+        return True  # Not critical, return True
+
+
+def verify_installation(logger: logging.Logger) -> bool:
+    """
+    Verify all NLP components are properly installed.
+    
+    Args:
+        logger: Logger instance
+        
+    Returns:
+        True if all components work, False otherwise
+    """
+    logger.info("Verifying NLP data installation...")
+    
+    success = True
+    
+    # Test NLTK
+    try:
+        import nltk
+        from nltk.corpus import stopwords
+        english_stopwords = stopwords.words('english')
+        logger.info(f"NLTK verification successful. Loaded {len(english_stopwords)} English stopwords")
+    except Exception as e:
+        logger.error(f"NLTK verification failed: {e}")
+        success = False
+    
+    # Test spaCy English model
+    try:
+        import spacy
+        nlp_en = spacy.load('en_core_web_sm')
+        doc = nlp_en("This is a test sentence.")
+        logger.info(f"spaCy English model verification successful. Processed {len(doc)} tokens")
+    except Exception as e:
+        logger.error(f"spaCy English model verification failed: {e}")
+        success = False
+    
+    # Test spaCy Chinese model (optional)
+    try:
+        import spacy
+        nlp_zh = spacy.load('zh_core_web_sm')
+        doc = nlp_zh("这是一个测试句子。")
+        logger.info(f"spaCy Chinese model verification successful. Processed {len(doc)} tokens")
+    except Exception as e:
+        logger.warning(f"spaCy Chinese model verification failed: {e}. This is optional.")
+
+    # Test spaCy PKUSeg model (optional)
+    try:
+        import spacy_pkuseg
+        seg = spacy_pkuseg.pkuseg()
+        result = list(seg.cut("这是一个测试句子"))
+        logger.info(f"spaCy PKUSeg model verification successful. Segmented: {result}")
+    except Exception as e:
+        logger.warning(f"spaCy PKUSeg model verification failed: {e}. This is optional.")
+
+    return success
+
+
+def main():
+    """Main function to download all required NLP data."""
+    logger = setup_logging()
+    logger.info("Starting AIECS NLP data download process...")
+    
+    success = True
+    
+    # Download NLTK data
+    if not download_nltk_data(logger):
+        success = False
+    
+    # Download spaCy English model
+    if not download_spacy_model('en_core_web_sm', logger):
+        success = False
+    
+    # Download spaCy Chinese model (optional)
+    if not download_spacy_model('zh_core_web_sm', logger):
+        logger.warning("Chinese model download failed, but this is optional")
+        # Don't mark as failure for Chinese model
+
+    # Download spaCy Chinese segmentation model (optional)
+    if not download_spacy_pkuseg_model(logger):
+        logger.warning("spaCy PKUSeg model download failed, but this is optional")
+        # Don't mark as failure for PKUSeg model
+    
+    # Check RAKE-NLTK (optional)
+    download_rake_nltk_data(logger)
+    
+    # Verify installation
+    if success and verify_installation(logger):
+        logger.info("✅ All NLP data downloaded and verified successfully!")
+        logger.info("AIECS ClassifierTool is ready to use.")
+        return 0
+    else:
+        logger.error("❌ Some NLP data downloads failed. Please check the logs above.")
+        logger.error("You may need to install missing packages or run this script again.")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())