rust-crate-pipeline 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

rust_crate_pipeline/ai_processing.py

@@ -166,8 +166,7 @@ class LLMEnricher:
         prompt: str,
         validation_func: Callable[[str], bool],
         temp: float = 0.2,
-        max_tokens: int = 256,
-        retries: int = 3
+        max_tokens: int = 256,        retries: int = 2  # Reduced default retries
     ) -> Optional[str]:
         """Run LLM with validation and automatic retry on failure"""
         for attempt in range(retries):
@@ -180,8 +179,11 @@ class LLMEnricher:
                 if result and validation_func(result):
                     return result
                 
-                # If we get here, validation failed
-                logging.warning(f"Validation failed on attempt {attempt+1}/{retries}. Retrying with modified parameters.")
+                # If we get here, validation failed - use debug level for early attempts
+                if attempt == retries - 1:
+                    logging.warning(f"Final validation attempt failed. Using best available result.")
+                else:
+                    logging.debug(f"Validation failed on attempt {attempt+1}/{retries}. Retrying with modified parameters.")
                 
                 # For the last attempt, simplify the prompt
                 if attempt == retries - 2:
@@ -190,11 +192,11 @@ class LLMEnricher:
             except Exception as e:
                 logging.error(f"Generation error on attempt {attempt+1}: {str(e)}")
             
-            # Backoff before retry
-            time.sleep(1.5 * (2 ** attempt))
+            # Reduced backoff to minimize waiting time
+            time.sleep(1.0 + (attempt * 0.5))
         
-        # If we exhaust all retries, return None
-        return None
+        # If we exhausted all retries, return the last result even if not perfect
+        return result if 'result' in locals() else None
 
     def simplify_prompt(self, prompt: str) -> str:
         """Simplify a prompt by removing examples and reducing context"""

rust_crate_pipeline/analysis.py

@@ -8,7 +8,8 @@ import tarfile
 import tempfile
 import subprocess
 import requests
-from datetime import datetime, relativedelta
+from datetime import datetime
+from dateutil.relativedelta import relativedelta
 from bs4 import BeautifulSoup
 from typing import Dict, Optional, List
 from .config import EnrichedCrate

rust_crate_pipeline/github_token_checker.py

@@ -0,0 +1,102 @@
+# github_token_checker.py
+"""
+GitHub Token Checker Module
+Lightweight version of the token checker for integration into the main pipeline.
+"""
+
+import os
+import sys
+import requests
+import logging
+
+def check_github_token_quick():
+    """Quick check if GitHub token is available and valid"""
+    token = os.getenv("GITHUB_TOKEN")
+    
+    if not token:
+        return False, "GITHUB_TOKEN environment variable not set"
+    
+    if len(token) < 20:
+        return False, "GITHUB_TOKEN seems too short - may be invalid"
+    
+    try:
+        # Quick API check
+        headers = {
+            "Accept": "application/vnd.github.v3+json",
+            "Authorization": f"token {token}"
+        }
+        
+        response = requests.get("https://api.github.com/rate_limit", headers=headers, timeout=10)
+        
+        if response.status_code == 200:
+            data = response.json()
+            remaining = data["resources"]["core"]["remaining"]
+            return True, f"Token valid, {remaining} API calls remaining"
+        elif response.status_code == 401:
+            return False, "GitHub token is invalid or expired"
+        else:
+            return False, f"GitHub API returned status code: {response.status_code}"
+            
+    except requests.exceptions.RequestException as e:
+        return False, f"Network error checking token: {str(e)}"
+    except Exception as e:
+        return False, f"Error checking token: {str(e)}"
+
+def prompt_for_token_setup():
+    """Prompt user to set up GitHub token"""
+    print("\n" + "="*60)
+    print("🔑 GitHub Token Required")
+    print("="*60)
+    print("\nThe Rust Crate Pipeline requires a GitHub Personal Access Token")
+    print("to access repository information and avoid rate limits.")
+    print("\n📋 Quick Setup:")
+    print("1. Get token: https://github.com/settings/tokens")
+    print("2. Required scopes: public_repo, read:user")
+    print("3. Set in environment:")
+    print("   export GITHUB_TOKEN=\"your_token_here\"")
+    print("\n🔧 Setup Scripts Available:")
+    print("   ./setup_github_token.sh    (Interactive setup)")
+    print("   python3 check_github_token.py    (Full verification)")
+    print("\n" + "="*60)
+    
+    # Ask if user wants to continue without token (limited functionality)
+    response = input("\nContinue without GitHub token? (y/N): ").strip().lower()
+    
+    if response in ['y', 'yes']:
+        print("⚠️  Running with limited GitHub API access (60 requests/hour)")
+        print("   You may encounter rate limit warnings.")
+        return True
+    else:
+        print("\n🛑 Please set up your GitHub token and try again.")
+        return False
+
+def check_and_setup_github_token():
+    """
+    Check GitHub token and prompt for setup if missing.
+    Returns True if should continue, False if should exit.
+    """
+    is_valid, message = check_github_token_quick()
+    
+    if is_valid:
+        logging.debug(f"GitHub token check: {message}")
+        return True
+    
+    # Token is missing or invalid
+    logging.warning(f"GitHub token issue: {message}")
+    
+    # Check if we're in a non-interactive environment
+    if not sys.stdin.isatty():
+        logging.error("GitHub token not configured and running in non-interactive mode")
+        logging.error("Set GITHUB_TOKEN environment variable before running")
+        return False
+    
+    # Interactive prompt
+    return prompt_for_token_setup()
+
+if __name__ == "__main__":
+    # Allow running this module directly for testing
+    is_valid, message = check_github_token_quick()
+    print(f"Token check: {'✅' if is_valid else '❌'} {message}")
+    
+    if not is_valid:
+        check_and_setup_github_token()

rust_crate_pipeline/main.py

@@ -8,6 +8,8 @@ import argparse
 from typing import Optional
 from .config import PipelineConfig
 from .pipeline import CrateDataPipeline
+from .production_config import setup_production_environment
+from .github_token_checker import check_and_setup_github_token
 
 def parse_arguments():
     """Parse command line arguments"""
@@ -21,6 +23,7 @@ Examples:
   python -m rust_crate_pipeline --batch-size 5     # Smaller batches
   python -m rust_crate_pipeline --output-dir ./data # Custom output directory
   python -m rust_crate_pipeline --log-level DEBUG   # Verbose logging
+  PRODUCTION=true python -m rust_crate_pipeline     # Production mode (quieter)
         """
     )
     
@@ -123,14 +126,31 @@ def check_disk_space():
         logging.warning("Low disk space! This may affect performance.")
 
 def main():
+    # Setup production environment first for optimal logging
+    prod_config = setup_production_environment()
+    
     args = parse_arguments()
     configure_logging(args.log_level)
     check_disk_space()
     
+    # Check GitHub token before proceeding
+    if not check_and_setup_github_token():
+        logging.error("GitHub token setup cancelled or failed. Exiting.")
+        sys.exit(1)
+    
     try:
         # Create config from command line arguments
         config_kwargs = {}
         
+        # Apply production optimizations if available
+        if prod_config:
+            config_kwargs.update({
+                'max_retries': prod_config.get('max_retries', 3),
+                'batch_size': prod_config.get('batch_size', 10),
+                'checkpoint_interval': prod_config.get('checkpoint_interval', 10),
+                'cache_ttl': prod_config.get('cache_ttl', 3600),
+            })
+        
         if args.batch_size:
             config_kwargs['batch_size'] = args.batch_size
         if args.workers:

rust_crate_pipeline/production_config.py

@@ -0,0 +1,76 @@
+# production_config.py
+"""
+Production configuration to reduce runtime warnings and optimize performance.
+This file contains settings that can be imported to minimize verbose logging
+and improve the user experience in production environments.
+"""
+
+import logging
+import os
+
+# Production logging configuration
+def configure_production_logging():
+    """Configure logging for production to reduce verbose warnings"""
+    
+    # Set up logging format
+    logging.basicConfig(
+        level=logging.INFO,  # Default to INFO level
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S'
+    )
+    
+    # Set specific loggers to less verbose levels
+    logging.getLogger('requests').setLevel(logging.WARNING)
+    logging.getLogger('urllib3').setLevel(logging.WARNING)
+    logging.getLogger('requests_cache').setLevel(logging.WARNING)
+    
+    # If PRODUCTION environment variable is set, be even quieter
+    if os.getenv('PRODUCTION', 'false').lower() == 'true':
+        logging.getLogger().setLevel(logging.WARNING)
+        logging.getLogger('rust_crate_pipeline').setLevel(logging.INFO)
+
+# Production-optimized settings
+PRODUCTION_SETTINGS = {
+    # Reduced retries to minimize warnings
+    'max_retries': 2,
+    'validation_retries': 2,
+    
+    # GitHub API management
+    'github_rate_limit_threshold': 100,
+    'github_critical_threshold': 50,
+    
+    # LLM settings  
+    'llm_timeout': 30,
+    'llm_max_attempts': 2,
+    
+    # Logging preferences
+    'quiet_mode': True,
+    'log_level': 'INFO',
+    
+    # Performance settings
+    'batch_size': 10,
+    'checkpoint_interval': 10,
+    'cache_ttl': 3600,
+}
+
+def get_production_config():
+    """Get production configuration dictionary"""
+    return PRODUCTION_SETTINGS.copy()
+
+def is_production():
+    """Check if running in production mode"""
+    return os.getenv('PRODUCTION', 'false').lower() == 'true'
+
+def setup_production_environment():
+    """Set up the complete production environment"""
+    configure_production_logging()
+    
+    # Set environment variables for quieter operation
+    os.environ.setdefault('PYTHONWARNINGS', 'ignore::UserWarning')
+    
+    if is_production():
+        print("🚀 Production mode enabled - optimized for minimal warnings")
+        return get_production_config()
+    else:
+        print("🔧 Development mode - full logging enabled")
+        return {}

rust_crate_pipeline/version.py

@@ -1,9 +1,19 @@
 """Version information for rust-crate-pipeline."""
 
-__version__ = "1.1.0"
+__version__ = "1.2.0"
 __version_info__ = tuple(int(x) for x in __version__.split("."))
 
 # Version history
+# 1.2.0 - Major release: Production-ready, cleaned codebase
+#         - Unified documentation into single comprehensive README
+#         - Removed all non-essential development and test files
+#         - Optimized for PyPI distribution and Docker deployment
+#         - Enhanced GitHub token integration and setup
+# 1.1.2 - Production release: Cleaned up non-essential files
+#         - Unified documentation into single README
+#         - Optimized for PyPI distribution
+# 1.1.1 - Bug fix: Added missing python-dateutil dependency
+#         - Fixed relativedelta import error
 # 1.1.0 - Updated author and contact information
 #         - Enhanced package configuration
 # 0.1.0 - Initial release