iam-policy-validator 1.8.0__py3-none-any.whl → 1.9.0__py3-none-any.whl

iam_validator/core/aws_service/__init__.py

@@ -0,0 +1,21 @@
+"""AWS Service Fetcher - Public API.
+
+This module provides functionality to fetch AWS service information from the AWS service
+reference API with advanced caching and performance features.
+
+Example usage:
+    async with AWSServiceFetcher() as fetcher:
+        services = await fetcher.fetch_services()
+        service_detail = await fetcher.fetch_service_by_name("s3")
+"""
+
+# Re-export main classes for public API
+from iam_validator.core.aws_service.fetcher import AWSServiceFetcher
+from iam_validator.core.aws_service.patterns import CompiledPatterns
+from iam_validator.core.aws_service.validators import ConditionKeyValidationResult
+
+__all__ = [
+    "AWSServiceFetcher",
+    "ConditionKeyValidationResult",
+    "CompiledPatterns",
+]

iam_validator/core/aws_service/cache.py

@@ -0,0 +1,108 @@
+"""Multi-layer caching for AWS service data.
+
+This module coordinates memory (LRU) and disk caching to optimize
+AWS service data retrieval performance.
+"""
+
+import logging
+from typing import Any
+
+from iam_validator.core.aws_service.storage import ServiceFileStorage
+from iam_validator.utils.cache import LRUCache
+
+logger = logging.getLogger(__name__)
+
+
+class ServiceCacheManager:
+    """Coordinates memory and disk caching for service data.
+
+    This class implements a two-tier caching strategy:
+    1. Fast in-memory LRU cache for frequently accessed data
+    2. Disk-based cache with TTL for persistence across runs
+
+    Cache lookup order:
+    1. Check memory cache (fastest)
+    2. Check disk cache (if enabled)
+    3. Return None if not found in either
+    """
+
+    def __init__(
+        self,
+        memory_cache_size: int = 256,
+        cache_ttl: int = 86400,
+        storage: ServiceFileStorage | None = None,
+    ) -> None:
+        """Initialize cache manager.
+
+        Args:
+            memory_cache_size: Maximum number of items in memory cache
+            cache_ttl: Cache time-to-live in seconds
+            storage: Optional storage backend for disk caching
+        """
+        self._memory_cache = LRUCache(maxsize=memory_cache_size, ttl=cache_ttl)
+        self._storage = storage
+
+    async def get(self, cache_key: str, url: str | None = None, base_url: str = "") -> Any | None:
+        """Get from memory cache, fallback to disk cache.
+
+        Args:
+            cache_key: Key to lookup in memory cache
+            url: Optional URL for disk cache lookup
+            base_url: Base URL for service reference API (used for disk cache path)
+
+        Returns:
+            Cached data if found, None otherwise
+        """
+        # Check memory cache first (fastest)
+        cached = await self._memory_cache.get(cache_key)
+        if cached is not None:
+            logger.debug(f"Memory cache hit for key: {cache_key}")
+            return cached
+
+        # Check disk cache if URL provided and storage available
+        if url and self._storage:
+            cached = self._storage.read_from_cache(url, base_url)
+            if cached is not None:
+                logger.debug(f"Disk cache hit for URL: {url}")
+                # Populate memory cache for faster future access
+                await self._memory_cache.set(cache_key, cached)
+                return cached
+
+        return None
+
+    async def set(
+        self, cache_key: str, value: Any, url: str | None = None, base_url: str = ""
+    ) -> None:
+        """Store in memory and optionally disk cache.
+
+        Args:
+            cache_key: Key to store in memory cache
+            value: Value to cache
+            url: Optional URL for disk cache storage
+            base_url: Base URL for service reference API (used for disk cache path)
+        """
+        # Always store in memory cache
+        await self._memory_cache.set(cache_key, value)
+
+        # Store in disk cache if URL provided and storage available
+        if url and self._storage:
+            self._storage.write_to_cache(url, value, base_url)
+
+    async def clear(self) -> None:
+        """Clear memory cache and optionally disk cache."""
+        await self._memory_cache.clear()
+
+        if self._storage:
+            self._storage.clear_disk_cache()
+
+        logger.info("Cleared all caches")
+
+    def get_stats(self) -> dict[str, int]:
+        """Get cache statistics.
+
+        Returns:
+            Dictionary with cache statistics
+        """
+        return {
+            "memory_cache_size": len(self._memory_cache.cache),
+        }

iam_validator/core/aws_service/client.py

@@ -0,0 +1,205 @@
+"""HTTP client for AWS Service Reference API.
+
+This module provides an async HTTP client with advanced features including
+connection pooling, request batching/coalescing, and retry logic.
+"""
+
+import asyncio
+import logging
+from typing import Any
+
+import httpx
+
+from iam_validator.core import constants
+
+logger = logging.getLogger(__name__)
+
+
+class AWSServiceClient:
+    """Async HTTP client with connection pooling and request coalescing.
+
+    This class handles all HTTP operations for fetching AWS service data,
+    including retry logic, request batching, and connection management.
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        timeout: float = constants.DEFAULT_HTTP_TIMEOUT_SECONDS,
+        retries: int = 3,
+        connection_pool_size: int = 50,
+        keepalive_connections: int = 20,
+    ) -> None:
+        """Initialize HTTP client.
+
+        Args:
+            base_url: Base URL for AWS service reference API
+            timeout: Request timeout in seconds
+            retries: Number of retries for failed requests
+            connection_pool_size: HTTP connection pool size
+            keepalive_connections: Number of keepalive connections
+        """
+        self.base_url = base_url
+        self.timeout = timeout
+        self.retries = retries
+        self.connection_pool_size = connection_pool_size
+        self.keepalive_connections = keepalive_connections
+
+        self._client: httpx.AsyncClient | None = None
+
+        # Batch request queue for request coalescing
+        self._batch_queue: dict[str, asyncio.Future[Any]] = {}
+        self._batch_lock = asyncio.Lock()
+
+    async def __aenter__(self) -> "AWSServiceClient":
+        """Setup httpx client with HTTP/2 and connection pooling."""
+        self._client = httpx.AsyncClient(
+            timeout=httpx.Timeout(self.timeout),
+            follow_redirects=True,
+            limits=httpx.Limits(
+                max_keepalive_connections=self.keepalive_connections,
+                max_connections=self.connection_pool_size,
+                keepalive_expiry=constants.DEFAULT_HTTP_TIMEOUT_SECONDS,
+            ),
+            http2=True,  # Enable HTTP/2 for multiplexing
+        )
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: Any,
+    ) -> None:
+        """Close HTTP client.
+
+        Args:
+            exc_type: Exception type (required by async context manager protocol)
+            exc_val: Exception value (required by async context manager protocol)
+            exc_tb: Exception traceback (required by async context manager protocol)
+        """
+        # Parameters required by protocol but not used in this implementation
+        del exc_type, exc_val, exc_tb
+
+        if self._client:
+            await self._client.aclose()
+            self._client = None
+
+    async def fetch(self, url: str) -> Any:
+        """Fetch data from URL with retry logic and batching.
+
+        This method implements request coalescing - if multiple coroutines
+        request the same URL simultaneously, only one HTTP request is made
+        and the result is shared.
+
+        Args:
+            url: URL to fetch
+
+        Returns:
+            Parsed JSON response data
+
+        Raises:
+            RuntimeError: If client is not initialized
+            ValueError: If response is not valid JSON or resource not found
+            Exception: If all retries fail
+        """
+        # First check: see if request is already in progress
+        existing_future = None
+        async with self._batch_lock:
+            if url in self._batch_queue:
+                existing_future = self._batch_queue[url]
+
+        # Wait for existing request outside the lock
+        if existing_future is not None:
+            logger.debug(f"Coalescing request for {url}")
+            return await existing_future
+
+        # Create new future for this request
+        loop = asyncio.get_event_loop()
+        future: asyncio.Future[Any] = loop.create_future()
+
+        # Second check: register future or use existing one (double-check pattern)
+        async with self._batch_lock:
+            if url in self._batch_queue:
+                # Another coroutine registered while we were creating the future
+                existing_future = self._batch_queue[url]
+            else:
+                # We're the first, register our future
+                self._batch_queue[url] = future
+
+        # If we found an existing future, wait for it
+        if existing_future is not None:
+            logger.debug(f"Coalescing request for {url} (late check)")
+            return await existing_future
+
+        # We're responsible for making the request
+        try:
+            # Actually make the request
+            result = await self._make_request(url)
+            if not future.done():
+                future.set_result(result)
+            return result
+        except Exception as e:
+            if not future.done():
+                future.set_exception(e)
+            raise
+        finally:
+            # Remove from queue
+            async with self._batch_lock:
+                self._batch_queue.pop(url, None)
+
+    async def _make_request(self, url: str) -> Any:
+        """Core HTTP request with exponential backoff.
+
+        Args:
+            url: URL to fetch
+
+        Returns:
+            Parsed JSON response data
+
+        Raises:
+            RuntimeError: If client is not initialized
+            ValueError: If response is not valid JSON or resource not found
+            Exception: If all retries fail
+        """
+        if not self._client:
+            raise RuntimeError("Client not initialized. Use as async context manager.")
+
+        last_exception: Exception | None = None
+
+        for attempt in range(self.retries):
+            try:
+                logger.debug(f"Fetching URL: {url} (attempt {attempt + 1})")
+                response = await self._client.get(url)
+                response.raise_for_status()
+
+                try:
+                    data = response.json()
+                    return data
+
+                except Exception as json_error:  # pylint: disable=broad-exception-caught
+                    logger.error(f"Failed to parse response as JSON: {json_error}")
+                    raise ValueError(
+                        f"Invalid JSON response from {url}: {json_error}"
+                    ) from json_error
+
+            except httpx.HTTPStatusError as e:
+                logger.error(f"HTTP error {e.response.status_code} for {url}")
+                if e.response.status_code == 404:
+                    raise ValueError(f"Service not found: {url}") from e
+                last_exception = e
+
+            except httpx.RequestError as e:
+                logger.error(f"Request error for {url}: {e}")
+                last_exception = e
+
+            except Exception as e:  # pylint: disable=broad-exception-caught
+                logger.error(f"Unexpected error for {url}: {e}")
+                last_exception = e
+
+            if attempt < self.retries - 1:
+                wait_time = 2**attempt
+                logger.info(f"Retrying in {wait_time} seconds...")
+                await asyncio.sleep(wait_time)
+
+        raise last_exception or Exception(f"Failed to fetch {url} after {self.retries} attempts")