reverse-diagrams 1.3.4__py3-none-any.whl → 2.0.0__py3-none-any.whl

src/utils/cache.py

@@ -0,0 +1,274 @@
+"""Caching system for AWS API responses."""
+import json
+import hashlib
+import logging
+from pathlib import Path
+from datetime import datetime, timedelta
+from typing import Optional, Dict, Any, Union
+import tempfile
+import os
+
+from ..config import get_config
+
+logger = logging.getLogger(__name__)
+
+
+class AWSDataCache:
+    """Cache for AWS API responses with TTL support."""
+    
+    def __init__(self, cache_dir: Optional[Path] = None, ttl_hours: int = 1):
+        """
+        Initialize cache.
+        
+        Args:
+            cache_dir: Cache directory path (optional, uses temp dir if None)
+            ttl_hours: Time to live in hours
+        """
+        if cache_dir is None:
+            cache_dir = Path(tempfile.gettempdir()) / "reverse_diagrams_cache"
+        
+        self.cache_dir = Path(cache_dir)
+        self.ttl = timedelta(hours=ttl_hours)
+        
+        # Create cache directory
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Set appropriate permissions
+        try:
+            os.chmod(self.cache_dir, 0o755)
+        except OSError:
+            logger.warning(f"Could not set permissions on cache directory: {self.cache_dir}")
+        
+        logger.debug(f"Cache initialized at {self.cache_dir} with TTL {ttl_hours}h")
+    
+    def _get_cache_key(self, operation: str, params: Dict[str, Any]) -> str:
+        """
+        Generate cache key from operation and parameters.
+        
+        Args:
+            operation: AWS operation name
+            params: Operation parameters
+            
+        Returns:
+            Cache key string
+        """
+        # Sort parameters for consistent hashing
+        param_str = json.dumps(params, sort_keys=True, default=str)
+        cache_string = f"{operation}:{param_str}"
+        
+        # Create hash
+        return hashlib.md5(cache_string.encode()).hexdigest()
+    
+    def get(self, operation: str, params: Dict[str, Any]) -> Optional[Any]:
+        """
+        Get cached data if still valid.
+        
+        Args:
+            operation: AWS operation name
+            params: Operation parameters
+            
+        Returns:
+            Cached data if valid, None otherwise
+        """
+        try:
+            cache_key = self._get_cache_key(operation, params)
+            cache_file = self.cache_dir / f"{cache_key}.json"
+            
+            if not cache_file.exists():
+                logger.debug(f"Cache miss for {operation}")
+                return None
+            
+            # Check if cache is still valid
+            stat = cache_file.stat()
+            cache_time = datetime.fromtimestamp(stat.st_mtime)
+            
+            if cache_time + self.ttl < datetime.now():
+                logger.debug(f"Cache expired for {operation}")
+                # Clean up expired cache
+                try:
+                    cache_file.unlink()
+                except OSError:
+                    pass
+                return None
+            
+            # Load and return cached data
+            with cache_file.open('r', encoding='utf-8') as f:
+                data = json.load(f)
+            
+            logger.debug(f"Cache hit for {operation}")
+            return data
+            
+        except Exception as e:
+            logger.warning(f"Error reading cache for {operation}: {e}")
+            return None
+    
+    def set(self, operation: str, params: Dict[str, Any], data: Any) -> None:
+        """
+        Cache data.
+        
+        Args:
+            operation: AWS operation name
+            params: Operation parameters
+            data: Data to cache
+        """
+        try:
+            cache_key = self._get_cache_key(operation, params)
+            cache_file = self.cache_dir / f"{cache_key}.json"
+            
+            # Write data to cache
+            with cache_file.open('w', encoding='utf-8') as f:
+                json.dump(data, f, indent=2, default=str)
+            
+            # Set appropriate permissions
+            try:
+                os.chmod(cache_file, 0o644)
+            except OSError:
+                pass
+            
+            logger.debug(f"Cached data for {operation}")
+            
+        except Exception as e:
+            logger.warning(f"Error writing cache for {operation}: {e}")
+    
+    def clear(self) -> None:
+        """Clear all cached data."""
+        try:
+            for cache_file in self.cache_dir.glob("*.json"):
+                cache_file.unlink()
+            logger.info("Cache cleared")
+        except Exception as e:
+            logger.warning(f"Error clearing cache: {e}")
+    
+    def clear_expired(self) -> int:
+        """
+        Clear expired cache entries.
+        
+        Returns:
+            Number of entries cleared
+        """
+        cleared = 0
+        try:
+            current_time = datetime.now()
+            
+            for cache_file in self.cache_dir.glob("*.json"):
+                try:
+                    stat = cache_file.stat()
+                    cache_time = datetime.fromtimestamp(stat.st_mtime)
+                    
+                    if cache_time + self.ttl < current_time:
+                        cache_file.unlink()
+                        cleared += 1
+                        
+                except OSError:
+                    continue
+            
+            if cleared > 0:
+                logger.info(f"Cleared {cleared} expired cache entries")
+                
+        except Exception as e:
+            logger.warning(f"Error clearing expired cache: {e}")
+        
+        return cleared
+    
+    def get_cache_info(self) -> Dict[str, Any]:
+        """
+        Get cache statistics.
+        
+        Returns:
+            Dictionary with cache information
+        """
+        try:
+            cache_files = list(self.cache_dir.glob("*.json"))
+            total_size = sum(f.stat().st_size for f in cache_files)
+            
+            # Count expired entries
+            current_time = datetime.now()
+            expired = 0
+            
+            for cache_file in cache_files:
+                try:
+                    stat = cache_file.stat()
+                    cache_time = datetime.fromtimestamp(stat.st_mtime)
+                    if cache_time + self.ttl < current_time:
+                        expired += 1
+                except OSError:
+                    continue
+            
+            return {
+                "cache_dir": str(self.cache_dir),
+                "total_entries": len(cache_files),
+                "expired_entries": expired,
+                "total_size_bytes": total_size,
+                "total_size_mb": round(total_size / (1024 * 1024), 2),
+                "ttl_hours": self.ttl.total_seconds() / 3600
+            }
+            
+        except Exception as e:
+            logger.warning(f"Error getting cache info: {e}")
+            return {"error": str(e)}
+
+
+# Global cache instance
+_cache: Optional[AWSDataCache] = None
+
+
+def get_cache() -> AWSDataCache:
+    """Get or create global cache instance."""
+    global _cache
+    
+    if _cache is None:
+        config = get_config()
+        if config.enable_caching:
+            _cache = AWSDataCache(ttl_hours=config.cache_ttl_hours)
+        else:
+            # Create a no-op cache that doesn't actually cache
+            _cache = NoOpCache()
+    
+    return _cache
+
+
+class NoOpCache:
+    """No-operation cache for when caching is disabled."""
+    
+    def get(self, operation: str, params: Dict[str, Any]) -> Optional[Any]:
+        return None
+    
+    def set(self, operation: str, params: Dict[str, Any], data: Any) -> None:
+        pass
+    
+    def clear(self) -> None:
+        pass
+    
+    def clear_expired(self) -> int:
+        return 0
+    
+    def get_cache_info(self) -> Dict[str, Any]:
+        return {"caching": "disabled"}
+
+
+def cached_aws_call(operation: str, params: Dict[str, Any], fetch_func) -> Any:
+    """
+    Decorator-like function for caching AWS API calls.
+    
+    Args:
+        operation: AWS operation name
+        params: Operation parameters
+        fetch_func: Function to call if cache miss
+        
+    Returns:
+        Cached or fresh data
+    """
+    cache = get_cache()
+    
+    # Try to get from cache first
+    cached_data = cache.get(operation, params)
+    if cached_data is not None:
+        return cached_data
+    
+    # Cache miss - fetch fresh data
+    fresh_data = fetch_func()
+    
+    # Cache the result
+    cache.set(operation, params, fresh_data)
+    
+    return fresh_data

src/utils/concurrent.py

@@ -0,0 +1,361 @@
+"""Concurrent processing utilities for AWS operations."""
+import asyncio
+import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import List, Dict, Any, Callable, Optional, TypeVar, Generic, Coroutine
+from dataclasses import dataclass
+import time
+
+from ..config import get_config
+from .progress import get_progress_tracker
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar('T')
+R = TypeVar('R')
+
+
+@dataclass
+class ProcessingResult(Generic[T, R]):
+    """Result of a concurrent processing operation."""
+    input_item: T
+    result: Optional[R] = None
+    error: Optional[Exception] = None
+    processing_time: float = 0.0
+    
+    @property
+    def success(self) -> bool:
+        """Check if processing was successful."""
+        return self.error is None
+    
+    @property
+    def failed(self) -> bool:
+        """Check if processing failed."""
+        return self.error is not None
+
+
+class ConcurrentAWSProcessor:
+    """Concurrent processor for AWS operations with progress tracking."""
+    
+    def __init__(self, max_workers: Optional[int] = None):
+        """
+        Initialize concurrent processor.
+        
+        Args:
+            max_workers: Maximum number of worker threads (optional)
+        """
+        config = get_config()
+        self.max_workers = max_workers or config.max_concurrent_workers
+        self.progress = get_progress_tracker()
+        
+        logger.info(f"Initialized concurrent processor with {self.max_workers} workers")
+    
+    def process_items(
+        self,
+        items: List[T],
+        processor_func: Callable[[T], R],
+        description: str = "Processing items",
+        fail_fast: bool = False
+    ) -> List[ProcessingResult[T, R]]:
+        """
+        Process items concurrently with progress tracking.
+        
+        Args:
+            items: List of items to process
+            processor_func: Function to process each item
+            description: Description for progress tracking
+            fail_fast: Whether to stop on first error
+            
+        Returns:
+            List of processing results
+        """
+        if not items:
+            logger.info("No items to process")
+            return []
+        
+        results: List[ProcessingResult[T, R]] = []
+        
+        with self.progress.track_operation(f"{description} ({len(items)} items)", total=len(items)) as task_id:
+            with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+                # Submit all tasks
+                future_to_item = {
+                    executor.submit(self._safe_process_item, item, processor_func): item
+                    for item in items
+                }
+                
+                # Process completed tasks
+                for future in as_completed(future_to_item):
+                    item = future_to_item[future]
+                    result = future.result()
+                    results.append(result)
+                    
+                    # Update progress
+                    self.progress.update_progress(task_id)
+                    
+                    # Handle errors
+                    if result.failed:
+                        logger.warning(f"Failed to process item {item}: {result.error}")
+                        if fail_fast:
+                            logger.error("Stopping processing due to fail_fast=True")
+                            # Cancel remaining futures
+                            for remaining_future in future_to_item:
+                                if not remaining_future.done():
+                                    remaining_future.cancel()
+                            break
+        
+        # Log summary
+        successful = sum(1 for r in results if r.success)
+        failed = len(results) - successful
+        total_time = sum(r.processing_time for r in results)
+        
+        logger.info(f"Processing complete: {successful} successful, {failed} failed, {total_time:.2f}s total")
+        
+        if failed > 0:
+            self.progress.show_warning(
+                f"Processing completed with {failed} failures",
+                f"Successfully processed {successful}/{len(items)} items"
+            )
+        else:
+            self.progress.show_success(
+                f"All {successful} items processed successfully",
+                f"Total processing time: {total_time:.2f}s"
+            )
+        
+        return results
+    
+    def _safe_process_item(self, item: T, processor_func: Callable[[T], R]) -> ProcessingResult[T, R]:
+        """
+        Safely process a single item with error handling and timing.
+        
+        Args:
+            item: Item to process
+            processor_func: Processing function
+            
+        Returns:
+            Processing result
+        """
+        start_time = time.time()
+        
+        try:
+            result = processor_func(item)
+            processing_time = time.time() - start_time
+            
+            return ProcessingResult(
+                input_item=item,
+                result=result,
+                processing_time=processing_time
+            )
+            
+        except Exception as e:
+            processing_time = time.time() - start_time
+            logger.debug(f"Error processing item {item}: {e}")
+            
+            return ProcessingResult(
+                input_item=item,
+                error=e,
+                processing_time=processing_time
+            )
+    
+    def process_accounts_concurrently(
+        self,
+        accounts: List[Dict[str, Any]],
+        processor_func: Callable[[Dict[str, Any]], Dict[str, Any]]
+    ) -> List[ProcessingResult[Dict[str, Any], Dict[str, Any]]]:
+        """
+        Process AWS accounts concurrently.
+        
+        Args:
+            accounts: List of account dictionaries
+            processor_func: Function to process each account
+            
+        Returns:
+            List of processing results
+        """
+        return self.process_items(
+            accounts,
+            processor_func,
+            description="Processing AWS accounts",
+            fail_fast=False
+        )
+    
+    def process_organizational_units_concurrently(
+        self,
+        ous: List[Dict[str, Any]],
+        processor_func: Callable[[Dict[str, Any]], Dict[str, Any]]
+    ) -> List[ProcessingResult[Dict[str, Any], Dict[str, Any]]]:
+        """
+        Process organizational units concurrently.
+        
+        Args:
+            ous: List of OU dictionaries
+            processor_func: Function to process each OU
+            
+        Returns:
+            List of processing results
+        """
+        return self.process_items(
+            ous,
+            processor_func,
+            description="Processing organizational units",
+            fail_fast=False
+        )
+    
+    def process_permission_sets_concurrently(
+        self,
+        permission_sets: List[str],
+        processor_func: Callable[[str], Dict[str, Any]]
+    ) -> List[ProcessingResult[str, Dict[str, Any]]]:
+        """
+        Process permission sets concurrently.
+        
+        Args:
+            permission_sets: List of permission set ARNs
+            processor_func: Function to process each permission set
+            
+        Returns:
+            List of processing results
+        """
+        return self.process_items(
+            permission_sets,
+            processor_func,
+            description="Processing permission sets",
+            fail_fast=False
+        )
+
+
+class AsyncAWSProcessor:
+    """Async processor for AWS operations (for future async AWS SDK support)."""
+    
+    def __init__(self, max_concurrent: Optional[int] = None):
+        """
+        Initialize async processor.
+        
+        Args:
+            max_concurrent: Maximum concurrent operations
+        """
+        config = get_config()
+        self.max_concurrent = max_concurrent or config.max_concurrent_workers
+        self.semaphore = asyncio.Semaphore(self.max_concurrent)
+        self.progress = get_progress_tracker()
+    
+    async def process_items_async(
+        self,
+        items: List[T],
+        processor_func: Callable[[T], Coroutine[Any, Any, R]],
+        description: str = "Processing items async"
+    ) -> List[ProcessingResult[T, R]]:
+        """
+        Process items asynchronously.
+        
+        Args:
+            items: List of items to process
+            processor_func: Async function to process each item
+            description: Description for progress tracking
+            
+        Returns:
+            List of processing results
+        """
+        if not items:
+            return []
+        
+        async def _process_with_semaphore(item: T) -> ProcessingResult[T, R]:
+            async with self.semaphore:
+                return await self._safe_process_item_async(item, processor_func)
+        
+        # Create tasks for all items
+        tasks = [_process_with_semaphore(item) for item in items]
+        
+        # Process with progress tracking
+        results = []
+        with self.progress.track_operation(f"{description} ({len(items)} items)", total=len(items)) as task_id:
+            for coro in asyncio.as_completed(tasks):
+                result = await coro
+                results.append(result)
+                self.progress.update_progress(task_id)
+        
+        return results
+    
+    async def _safe_process_item_async(
+        self,
+        item: T,
+        processor_func: Callable[[T], Coroutine[Any, Any, R]]
+    ) -> ProcessingResult[T, R]:
+        """
+        Safely process a single item asynchronously.
+        
+        Args:
+            item: Item to process
+            processor_func: Async processing function
+            
+        Returns:
+            Processing result
+        """
+        start_time = time.time()
+        
+        try:
+            result = await processor_func(item)
+            processing_time = time.time() - start_time
+            
+            return ProcessingResult(
+                input_item=item,
+                result=result,
+                processing_time=processing_time
+            )
+            
+        except Exception as e:
+            processing_time = time.time() - start_time
+            
+            return ProcessingResult(
+                input_item=item,
+                error=e,
+                processing_time=processing_time
+            )
+
+
+# Global processor instances
+_concurrent_processor: Optional[ConcurrentAWSProcessor] = None
+_async_processor: Optional[AsyncAWSProcessor] = None
+
+
+def get_concurrent_processor() -> ConcurrentAWSProcessor:
+    """Get or create global concurrent processor instance."""
+    global _concurrent_processor
+    if _concurrent_processor is None:
+        _concurrent_processor = ConcurrentAWSProcessor()
+    return _concurrent_processor
+
+
+def get_async_processor() -> AsyncAWSProcessor:
+    """Get or create global async processor instance."""
+    global _async_processor
+    if _async_processor is None:
+        _async_processor = AsyncAWSProcessor()
+    return _async_processor
+
+
+def process_concurrently(
+    items: List[T],
+    processor_func: Callable[[T], R],
+    description: str = "Processing items",
+    max_workers: Optional[int] = None,
+    fail_fast: bool = False
+) -> List[ProcessingResult[T, R]]:
+    """
+    Convenience function for concurrent processing.
+    
+    Args:
+        items: List of items to process
+        processor_func: Function to process each item
+        description: Description for progress tracking
+        max_workers: Maximum number of workers (optional)
+        fail_fast: Whether to stop on first error
+        
+    Returns:
+        List of processing results
+    """
+    if max_workers:
+        processor = ConcurrentAWSProcessor(max_workers=max_workers)
+    else:
+        processor = get_concurrent_processor()
+    
+    return processor.process_items(items, processor_func, description, fail_fast)