aiecs 1.0.1__py3-none-any.whl → 1.7.6__py3-none-any.whl

aiecs/scripts/{download_nlp_data.py → dependance_check/download_nlp_data.py}

@@ -3,13 +3,16 @@
 Automated script to download required NLP data for AIECS ClassifierTool.
 
 This script downloads:
-1. NLTK stopwords data package for keyword extraction
+1. NLTK stopwords data package for keyword extraction (to environment-specific location)
 2. spaCy English model (en_core_web_sm) for text processing
 3. spaCy Chinese model (zh_core_web_sm) for Chinese text processing
+
+All NLP data is downloaded to the current Poetry/virtual environment to ensure
+environment isolation. NLTK data is stored in <env_path>/nltk_data/ directory.
 """
 
-import os
 import sys
+import os
 import subprocess
 import logging
 from pathlib import Path
@@ -20,11 +23,11 @@ def setup_logging():
     """Setup logging configuration."""
     logging.basicConfig(
         level=logging.INFO,
-        format='%(asctime)s - %(levelname)s - %(message)s',
+        format="%(asctime)s - %(levelname)s - %(message)s",
         handlers=[
             logging.StreamHandler(sys.stdout),
-            logging.FileHandler('nlp_data_download.log')
-        ]
+            logging.FileHandler("nlp_data_download.log"),
+        ],
     )
     return logging.getLogger(__name__)
 
@@ -32,22 +35,17 @@ def setup_logging():
 def run_command(cmd: List[str], logger: logging.Logger) -> Tuple[bool, str]:
     """
     Run a shell command and return success status and output.
-    
+
     Args:
         cmd: List of command arguments
         logger: Logger instance
-    
+
     Returns:
         Tuple of (success, output)
     """
     try:
         logger.info(f"Running command: {' '.join(cmd)}")
-        result = subprocess.run(
-            cmd, 
-            capture_output=True, 
-            text=True, 
-            check=True
-        )
+        result = subprocess.run(cmd, capture_output=True, text=True, check=True)
         logger.info(f"Command succeeded: {result.stdout}")
         return True, result.stdout
     except subprocess.CalledProcessError as e:
@@ -60,14 +58,54 @@ def run_command(cmd: List[str], logger: logging.Logger) -> Tuple[bool, str]:
         return False, error_msg
 
 
+def get_environment_path(logger: logging.Logger) -> Optional[Path]:
+    """
+    Get the path to the current Python environment (virtual environment or Poetry environment).
+
+    Args:
+        logger: Logger instance
+
+    Returns:
+        Path to the environment if found, None otherwise
+    """
+    # Check VIRTUAL_ENV environment variable first (common for venv/virtualenv)
+    venv_path = os.environ.get("VIRTUAL_ENV")
+    if venv_path:
+        env_path = Path(venv_path)
+        if env_path.exists():
+            logger.info(f"Found virtual environment via VIRTUAL_ENV: {env_path}")
+            return env_path
+
+    # Check sys.prefix - this points to the virtual environment if we're in one
+    # In a virtual environment, sys.prefix != sys.base_prefix
+    if sys.prefix != sys.base_prefix:
+        env_path = Path(sys.prefix)
+        if env_path.exists():
+            logger.info(f"Found virtual environment via sys.prefix: {env_path}")
+            return env_path
+
+    # Check if we're in a Poetry environment by checking sys.executable path
+    # Poetry environments are typically in ~/.cache/pypoetry/virtualenvs/ or similar
+    exec_path = Path(sys.executable)
+    if "pypoetry" in str(exec_path) or exec_path.parts[-3:-1] == ("bin", "python"):
+        # Try to find the environment root (go up from bin/python)
+        potential_env = exec_path.parent.parent
+        if potential_env.exists() and (potential_env / "pyvenv.cfg").exists():
+            logger.info(f"Found Poetry/virtual environment: {potential_env}")
+            return potential_env
+
+    logger.warning("No virtual environment detected. NLTK data will be downloaded to user directory.")
+    return None
+
+
 def check_python_package(package_name: str, logger: logging.Logger) -> bool:
     """
     Check if a Python package is installed.
-    
+
     Args:
         package_name: Name of the package to check
         logger: Logger instance
-        
+
     Returns:
         True if package is installed, False otherwise
     """
@@ -82,31 +120,53 @@ def check_python_package(package_name: str, logger: logging.Logger) -> bool:
 
 def download_nltk_data(logger: logging.Logger) -> bool:
     """
-    Download required NLTK data packages.
-    
+    Download required NLTK data packages to the current environment.
+
     Args:
         logger: Logger instance
-    
+
     Returns:
         True if successful, False otherwise
     """
     logger.info("Starting NLTK data download...")
-    
-    if not check_python_package('nltk', logger):
+
+    if not check_python_package("nltk", logger):
         logger.error("NLTK is not installed. Please install it first with: pip install nltk")
         return False
-    
+
+    # Get the environment path to store NLTK data environment-specifically
+    env_path = get_environment_path(logger)
+    nltk_data_path: Optional[Path] = None
+    original_nltk_data = os.environ.get("NLTK_DATA")
+
+    # Set NLTK_DATA environment variable BEFORE importing nltk
+    # This ensures nltk.data.path is initialized with the correct path
+    if env_path:
+        # Create nltk_data directory in the environment
+        nltk_data_path = env_path / "nltk_data"
+        nltk_data_path.mkdir(parents=True, exist_ok=True)
+        # Set NLTK_DATA environment variable BEFORE importing nltk
+        os.environ["NLTK_DATA"] = str(nltk_data_path)
+        logger.info(f"Using environment-specific NLTK data directory: {nltk_data_path}")
+    else:
+        logger.info("No virtual environment detected. Using default NLTK data location (~/nltk_data)")
+
     try:
-        import nltk
-        
+        import nltk  # type: ignore[import-untyped]
+
+        # Ensure nltk.data.path includes our environment-specific path
+        if nltk_data_path and str(nltk_data_path) not in nltk.data.path:
+            nltk.data.path.insert(0, str(nltk_data_path))
+
         # Download required NLTK data
         packages_to_download = [
-            'stopwords',
-            'punkt',
-            'wordnet',
-            'averaged_perceptron_tagger'
+            "stopwords",
+            "punkt",
+            "punkt_tab",  # Added for RAKE-NLTK compatibility
+            "wordnet",
+            "averaged_perceptron_tagger",
         ]
-        
+
         for package in packages_to_download:
             try:
                 logger.info(f"Downloading NLTK package: {package}")
@@ -114,36 +174,50 @@ def download_nltk_data(logger: logging.Logger) -> bool:
                 logger.info(f"Successfully downloaded NLTK package: {package}")
             except Exception as e:
                 logger.error(f"Failed to download NLTK package {package}: {e}")
+                # Restore original NLTK_DATA if we changed it
+                if original_nltk_data is not None:
+                    os.environ["NLTK_DATA"] = original_nltk_data
+                elif nltk_data_path:
+                    os.environ.pop("NLTK_DATA", None)
                 return False
-                
+
         logger.info("All NLTK data packages downloaded successfully")
+        if nltk_data_path:
+            logger.info(f"NLTK data is stored in environment-specific location: {nltk_data_path}")
+            logger.info("Note: Set NLTK_DATA environment variable to this path if needed in other scripts")
         return True
-        
+
     except Exception as e:
         logger.error(f"Error downloading NLTK data: {e}")
+        # Restore original NLTK_DATA if we changed it
+        if original_nltk_data is not None:
+            os.environ["NLTK_DATA"] = original_nltk_data
+        elif nltk_data_path:
+            os.environ.pop("NLTK_DATA", None)
         return False
 
 
 def download_spacy_model(model_name: str, logger: logging.Logger) -> bool:
     """
     Download a spaCy model.
-    
+
     Args:
         model_name: Name of the spaCy model to download
         logger: Logger instance
-    
+
     Returns:
         True if successful, False otherwise
     """
     logger.info(f"Starting spaCy model download: {model_name}")
-    
-    if not check_python_package('spacy', logger):
+
+    if not check_python_package("spacy", logger):
         logger.error("spaCy is not installed. Please install it first with: pip install spacy")
         return False
-    
+
     # Check if model is already installed
     try:
         import spacy
+
         spacy.load(model_name)
         logger.info(f"spaCy model {model_name} is already installed")
         return True
@@ -153,17 +227,18 @@ def download_spacy_model(model_name: str, logger: logging.Logger) -> bool:
     except Exception as e:
         logger.error(f"Error checking spaCy model {model_name}: {e}")
         return False
-    
+
     # Download the model
     cmd = [sys.executable, "-m", "spacy", "download", model_name]
     success, output = run_command(cmd, logger)
-    
+
     if success:
         logger.info(f"Successfully downloaded spaCy model: {model_name}")
-        
+
         # Verify the model can be loaded
         try:
             import spacy
+
             spacy.load(model_name)
             logger.info(f"Verified spaCy model {model_name} can be loaded")
             return True
@@ -173,7 +248,7 @@ def download_spacy_model(model_name: str, logger: logging.Logger) -> bool:
     else:
         logger.error(f"Failed to download spaCy model {model_name}: {output}")
         return False
-    
+
 
 def download_spacy_pkuseg_model(logger: logging.Logger) -> bool:
     """
@@ -187,12 +262,12 @@ def download_spacy_pkuseg_model(logger: logging.Logger) -> bool:
     """
     logger.info("Starting spaCy PKUSeg model installation...")
 
-    if not check_python_package('spacy', logger):
+    if not check_python_package("spacy", logger):
         logger.error("spaCy is not installed. Please install it first with: pip install spacy")
         return False
 
     # Check if spacy_pkuseg is already installed
-    if check_python_package('spacy_pkuseg', logger):
+    if check_python_package("spacy_pkuseg", logger):
         logger.info("spacy_pkuseg is already installed")
         return True
 
@@ -205,7 +280,8 @@ def download_spacy_pkuseg_model(logger: logging.Logger) -> bool:
 
         # Verify the package can be imported
         try:
-            import spacy_pkuseg
+            import spacy_pkuseg  # type: ignore[import-untyped]
+
             logger.info("Verified spacy_pkuseg can be imported")
 
             # Test basic functionality
@@ -224,25 +300,26 @@ def download_spacy_pkuseg_model(logger: logging.Logger) -> bool:
 def download_rake_nltk_data(logger: logging.Logger) -> bool:
     """
     Ensure RAKE-NLTK has required data.
-    
+
     Args:
         logger: Logger instance
-    
+
     Returns:
         True if successful, False otherwise
     """
     logger.info("Checking RAKE-NLTK data...")
-    
-    if not check_python_package('rake_nltk', logger):
+
+    if not check_python_package("rake_nltk", logger):
         logger.warning("RAKE-NLTK is not installed. This is optional for English keyword extraction.")
         return True  # Not critical, return True
-    
+
     try:
-        from rake_nltk import Rake
+        from rake_nltk import Rake  # type: ignore[import-untyped]
+
         # Test RAKE functionality
         rake = Rake()
         rake.extract_keywords_from_text("This is a test sentence for RAKE.")
-        keywords = rake.get_ranked_phrases()
+        rake.get_ranked_phrases()
         logger.info("RAKE-NLTK is working correctly")
         return True
     except Exception as e:
@@ -253,41 +330,52 @@ def download_rake_nltk_data(logger: logging.Logger) -> bool:
 def verify_installation(logger: logging.Logger) -> bool:
     """
     Verify all NLP components are properly installed.
-    
+
     Args:
         logger: Logger instance
-        
+
     Returns:
         True if all components work, False otherwise
     """
     logger.info("Verifying NLP data installation...")
-    
+
     success = True
-    
+
     # Test NLTK
     try:
-        import nltk
-        from nltk.corpus import stopwords
-        english_stopwords = stopwords.words('english')
+        # Ensure we check the environment-specific NLTK data location if it exists
+        env_path = get_environment_path(logger)
+        if env_path:
+            nltk_data_path = env_path / "nltk_data"
+            if nltk_data_path.exists():
+                os.environ["NLTK_DATA"] = str(nltk_data_path)
+
+        from nltk.corpus import stopwords  # type: ignore[import-untyped]
+
+        english_stopwords = stopwords.words("english")
         logger.info(f"NLTK verification successful. Loaded {len(english_stopwords)} English stopwords")
+        if env_path and (env_path / "nltk_data").exists():
+            logger.info(f"NLTK data is located in environment: {env_path / 'nltk_data'}")
     except Exception as e:
         logger.error(f"NLTK verification failed: {e}")
         success = False
-    
+
     # Test spaCy English model
     try:
         import spacy
-        nlp_en = spacy.load('en_core_web_sm')
+
+        nlp_en = spacy.load("en_core_web_sm")
         doc = nlp_en("This is a test sentence.")
         logger.info(f"spaCy English model verification successful. Processed {len(doc)} tokens")
     except Exception as e:
         logger.error(f"spaCy English model verification failed: {e}")
         success = False
-    
+
     # Test spaCy Chinese model (optional)
     try:
         import spacy
-        nlp_zh = spacy.load('zh_core_web_sm')
+
+        nlp_zh = spacy.load("zh_core_web_sm")
         doc = nlp_zh("这是一个测试句子。")
         logger.info(f"spaCy Chinese model verification successful. Processed {len(doc)} tokens")
     except Exception as e:
@@ -296,6 +384,7 @@ def verify_installation(logger: logging.Logger) -> bool:
     # Test spaCy PKUSeg model (optional)
     try:
         import spacy_pkuseg
+
         seg = spacy_pkuseg.pkuseg()
         result = list(seg.cut("这是一个测试句子"))
         logger.info(f"spaCy PKUSeg model verification successful. Segmented: {result}")
@@ -305,23 +394,23 @@ def verify_installation(logger: logging.Logger) -> bool:
     return success
 
 
-def main():
-    """Main function to download all required NLP data."""
+def download_all_nlp_data():
+    """Download all required NLP data."""
     logger = setup_logging()
     logger.info("Starting AIECS NLP data download process...")
-    
+
     success = True
-    
+
     # Download NLTK data
     if not download_nltk_data(logger):
         success = False
-    
+
     # Download spaCy English model
-    if not download_spacy_model('en_core_web_sm', logger):
+    if not download_spacy_model("en_core_web_sm", logger):
         success = False
-    
+
     # Download spaCy Chinese model (optional)
-    if not download_spacy_model('zh_core_web_sm', logger):
+    if not download_spacy_model("zh_core_web_sm", logger):
         logger.warning("Chinese model download failed, but this is optional")
         # Don't mark as failure for Chinese model
 
@@ -329,10 +418,10 @@ def main():
     if not download_spacy_pkuseg_model(logger):
         logger.warning("spaCy PKUSeg model download failed, but this is optional")
         # Don't mark as failure for PKUSeg model
-    
+
     # Check RAKE-NLTK (optional)
     download_rake_nltk_data(logger)
-    
+
     # Verify installation
     if success and verify_installation(logger):
         logger.info("✅ All NLP data downloaded and verified successfully!")
@@ -344,5 +433,48 @@ def main():
         return 1
 
 
+def main():
+    """Main entry point with argument parsing."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Download NLP data for AIECS tools",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Show this help message
+  aiecs-download-nlp-data --help
+
+  # Download all NLP data
+  aiecs-download-nlp-data --download
+  aiecs-download-nlp-data -d
+
+NLP Data Includes:
+  - NLTK packages: stopwords, punkt, wordnet, averaged_perceptron_tagger
+  - spaCy models: en_core_web_sm (English), zh_core_web_sm (Chinese, optional)
+  - spaCy PKUSeg model (Chinese segmentation, optional)
+  - RAKE-NLTK data (keyword extraction, optional)
+        """,
+    )
+
+    parser.add_argument(
+        "-d",
+        "--download",
+        action="store_true",
+        help="Download all NLP data packages",
+    )
+
+    args = parser.parse_args()
+
+    # If no arguments provided, show help
+    if not args.download:
+        parser.print_help()
+        print("\n⚠️  No action specified. Use --download or -d to download NLP data.")
+        return 0
+
+    # Execute download
+    return download_all_nlp_data()
+
+
 if __name__ == "__main__":
-    sys.exit(main())
+    sys.exit(main())

aiecs/scripts/dependance_patch/__init__.py

@@ -0,0 +1,7 @@
+"""
+依赖补丁工具
+
+提供各种依赖库的补丁修复功能。
+"""
+
+__all__ = []

aiecs/scripts/dependance_patch/fix_weasel/__init__.py

@@ -0,0 +1,11 @@
+"""
+Weasel 库补丁工具
+
+修复 Weasel 库的验证器问题。
+"""
+
+from .fix_weasel_validator import main as fix_weasel_validator_main
+
+__all__ = [
+    "fix_weasel_validator_main",
+]

aiecs/scripts/{fix_weasel_validator.py → dependance_patch/fix_weasel/fix_weasel_validator.py}

@@ -6,19 +6,20 @@ This script patches the weasel schemas.py file to add allow_reuse=True to duplic
 
 import os
 import sys
-import subprocess
 import shutil
 from datetime import datetime
 import re
 
+
 def get_weasel_path():
     """Get the weasel package path in the current Python environment."""
     try:
-        import weasel
+        import weasel  # type: ignore[import-untyped]
         import inspect
+
         weasel_file = inspect.getfile(weasel)
         weasel_dir = os.path.dirname(weasel_file)
-        return os.path.join(weasel_dir, 'schemas.py')
+        return os.path.join(weasel_dir, "schemas.py")
     except ImportError:
         print("❌ Error: weasel package not found")
         print("Please install aiecs with all dependencies")
@@ -27,6 +28,7 @@ def get_weasel_path():
         print(f"❌ Error finding weasel package: {e}")
         return None
 
+
 def backup_file(file_path):
     """Create a backup of the file."""
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
@@ -34,17 +36,18 @@ def backup_file(file_path):
     shutil.copy2(file_path, backup_path)
     return backup_path
 
+
 def fix_weasel_schemas(schemas_file_path):
     """Fix the weasel schemas.py file by adding allow_reuse=True to validators."""
 
     print(f"📁 Processing file: {schemas_file_path}")
 
     # Read the original file
-    with open(schemas_file_path, 'r', encoding='utf-8') as f:
+    with open(schemas_file_path, "r", encoding="utf-8") as f:
         content = f.read()
 
     # Check if already patched
-    if 'allow_reuse=True' in content:
+    if "allow_reuse=True" in content:
         print("✅ File already patched with allow_reuse=True")
         return True
 
@@ -53,19 +56,21 @@ def fix_weasel_schemas(schemas_file_path):
     print(f"💾 Created backup at: {backup_path}")
 
     # Show current problematic area
-    lines = content.split('\n')
+    lines = content.split("\n")
     print("\n📖 Current content around line 89:")
     for i, line in enumerate(lines[84:94], 85):
         print(f"{i:3d} | {line}")
 
-    # Pattern to match both @validator and @root_validator decorators without allow_reuse
-    validator_pattern = r'(@(?:root_)?validator\([^)]*)\)(?!\s*,\s*allow_reuse=True)'
+    # Pattern to match both @validator and @root_validator decorators without
+    # allow_reuse
+    validator_pattern = r"(@(?:root_)?validator\([^)]*)\)(?!\s*,\s*allow_reuse=True)"
 
-    # Replace @validator(...) or @root_validator(...) with allow_reuse=True if not already present
+    # Replace @validator(...) or @root_validator(...) with allow_reuse=True if
+    # not already present
     def replace_validator(match):
         validator_call = match.group(1)
         # Check if allow_reuse is already in the parameters
-        if 'allow_reuse' in validator_call:
+        if "allow_reuse" in validator_call:
             return match.group(0)  # Return unchanged
         else:
             return f"{validator_call}, allow_reuse=True)"
@@ -74,23 +79,24 @@ def fix_weasel_schemas(schemas_file_path):
     fixed_content = re.sub(validator_pattern, replace_validator, content)
 
     # Write the fixed content back
-    with open(schemas_file_path, 'w', encoding='utf-8') as f:
+    with open(schemas_file_path, "w", encoding="utf-8") as f:
         f.write(fixed_content)
 
     # Show the fixed content
-    fixed_lines = fixed_content.split('\n')
+    fixed_lines = fixed_content.split("\n")
     print("\n📖 Patched content around line 89:")
     for i, line in enumerate(fixed_lines[84:94], 85):
         print(f"{i:3d} | {line}")
 
     # Verify the fix
-    if 'allow_reuse=True' in fixed_content:
+    if "allow_reuse=True" in fixed_content:
         print("✅ Verification successful: allow_reuse=True found in file")
         return True
     else:
         print("⚠️  Warning: allow_reuse=True not found after patching")
         return False
 
+
 def main():
     """Main function to execute the patch."""
     print("🔧 Starting weasel library patch for duplicate validator function...")
@@ -112,10 +118,11 @@ def main():
     if success:
         print("\n🎉 Weasel library patch completed successfully!")
         print("\nYou can now run your tests again.")
-        print(f"\nIf you need to revert the changes, restore from the backup file.")
+        print("\nIf you need to revert the changes, restore from the backup file.")
     else:
         print("\n❌ Patch may not have been applied correctly. Please check manually.")
         sys.exit(1)
 
+
 if __name__ == "__main__":
     main()

aiecs/scripts/{patch_weasel_library.sh → dependance_patch/fix_weasel/patch_weasel_library.sh}

@@ -149,7 +149,7 @@ test_fix() {
 import sys
 sys.path.insert(0, '.')
 try:
-    from app.tools.task_tools.research_tool import *
+    from aiecs.tools.task_tools.research_tool import *
     print('✅ Import successful - fix appears to work!')
 except Exception as e:
     print(f'❌ Import still fails: {e}')

aiecs/scripts/knowledge_graph/__init__.py

@@ -0,0 +1,3 @@
+"""
+Knowledge Graph Scripts
+"""