nui-python-shared-utils 1.3.0__py3-none-any.whl

nui_lambda_shared_utils/__init__.py

@@ -0,0 +1,252 @@
+"""
+Enterprise-grade utilities for AWS Lambda functions with Slack, Elasticsearch, and monitoring integrations.
+"""
+
+# Configuration system
+from .config import (
+    Config,
+    get_config,
+    set_config,
+    configure,
+    get_es_host,
+    get_es_credentials_secret,
+    get_db_credentials_secret,
+    get_slack_credentials_secret,
+)
+
+# Core utilities
+from .secrets_helper import (
+    get_secret,
+    get_database_credentials,
+    get_elasticsearch_credentials,
+    get_slack_credentials,
+    get_api_key,
+    clear_cache,
+)
+
+# Common utilities
+from .utils import (
+    resolve_config_value,
+    create_aws_client,
+    handle_client_errors,
+    merge_dimensions,
+    validate_required_param,
+)
+
+# Base client architecture
+from .base_client import BaseClient, ServiceHealthMixin, RetryableOperationMixin
+
+# Client implementations - only fail if actually used
+try:
+    from .slack_client import SlackClient
+except ImportError:
+    SlackClient = None  # type: ignore
+
+try:
+    from .es_client import ElasticsearchClient
+except ImportError:
+    ElasticsearchClient = None  # type: ignore
+
+try:
+    from .db_client import DatabaseClient, PostgreSQLClient, get_pool_stats
+except ImportError:
+    DatabaseClient = None  # type: ignore
+    PostgreSQLClient = None  # type: ignore
+    get_pool_stats = None  # type: ignore
+
+from .timezone import nz_time, format_nz_time
+
+# Slack formatting utilities (no external dependencies)
+from .slack_formatter import (
+    SlackBlockBuilder,
+    format_currency,
+    format_percentage,
+    format_number,
+    format_nz_time as format_nz_time_slack,
+    format_date_range,
+    format_daily_header,
+    format_weekly_header,
+    format_error_alert,
+    SEVERITY_EMOJI,
+    STATUS_EMOJI,
+)
+
+# ES query builder - optional import
+try:
+    from .es_query_builder import (
+        ESQueryBuilder,
+        build_error_rate_query,
+        build_top_errors_query,
+        build_response_time_query,
+        build_service_volume_query,
+        build_user_activity_query,
+        build_pattern_detection_query,
+        build_tender_participant_query,
+    )
+except ImportError:
+    ESQueryBuilder = None  # type: ignore
+    build_error_rate_query = None  # type: ignore
+    build_top_errors_query = None  # type: ignore
+    build_response_time_query = None  # type: ignore
+    build_service_volume_query = None  # type: ignore
+    build_user_activity_query = None  # type: ignore
+    build_pattern_detection_query = None  # type: ignore
+    build_tender_participant_query = None  # type: ignore
+from .error_handler import (
+    RetryableError,
+    NonRetryableError,
+    ErrorPatternMatcher,
+    ErrorAggregator,
+    with_retry,
+    retry_on_network_error,
+    retry_on_db_error,
+    retry_on_es_error,
+    handle_lambda_error,
+    categorize_retryable_error,
+)
+from .cloudwatch_metrics import (
+    MetricsPublisher,
+    MetricAggregator,
+    StandardMetrics,
+    TimedMetric,
+    track_lambda_performance,
+    create_service_dimensions,
+    publish_health_metric,
+)
+
+# AWS Powertools integration - optional import
+try:
+    from .powertools_helpers import get_powertools_logger, powertools_handler
+except ImportError:
+    get_powertools_logger = None  # type: ignore
+    powertools_handler = None  # type: ignore
+
+# Log processing utilities (no external dependencies)
+from .log_processors import (
+    CloudWatchLogEvent,
+    CloudWatchLogsData,
+    derive_index_name,
+    extract_cloudwatch_logs_from_kinesis,
+)
+
+# Lambda context helpers (no external dependencies)
+from .lambda_helpers import get_lambda_environment_info
+
+# JWT authentication - optional import
+try:
+    from .jwt_auth import (
+        validate_jwt,
+        require_auth,
+        check_auth,
+        get_jwt_public_key,
+        JWTValidationError,
+        AuthenticationError,
+    )
+except ImportError:
+    validate_jwt = None  # type: ignore
+    require_auth = None  # type: ignore
+    check_auth = None  # type: ignore
+    get_jwt_public_key = None  # type: ignore
+    JWTValidationError = None  # type: ignore
+    AuthenticationError = None  # type: ignore
+
+# Slack setup utilities (for CLI usage) - optional import
+try:
+    from . import slack_setup
+except ImportError:
+    slack_setup = None  # type: ignore
+
+__all__ = [
+    # Configuration system
+    "Config",
+    "get_config",
+    "set_config",
+    "configure",
+    "get_es_host",
+    "get_es_credentials_secret",
+    "get_db_credentials_secret",
+    "get_slack_credentials_secret",
+    # Core utilities
+    "get_secret",
+    "get_database_credentials",
+    "get_elasticsearch_credentials",
+    "get_slack_credentials",
+    "get_api_key",
+    "clear_cache",
+    # Common utilities
+    "resolve_config_value",
+    "create_aws_client",
+    "handle_client_errors",
+    "merge_dimensions",
+    "validate_required_param",
+    # Base client architecture
+    "BaseClient",
+    "ServiceHealthMixin",
+    "RetryableOperationMixin",
+    # Client implementations
+    "SlackClient",
+    "ElasticsearchClient",
+    "DatabaseClient",
+    "PostgreSQLClient",
+    "get_pool_stats",  # Legacy compatibility (None)
+    "nz_time",
+    "format_nz_time",
+    "slack_setup",
+    # Slack formatting
+    "SlackBlockBuilder",
+    "format_currency",
+    "format_percentage",
+    "format_number",
+    "format_nz_time_slack",
+    "format_date_range",
+    "format_daily_header",
+    "format_weekly_header",
+    "format_error_alert",
+    "SEVERITY_EMOJI",
+    "STATUS_EMOJI",
+    # ES query building
+    "ESQueryBuilder",
+    "build_error_rate_query",
+    "build_top_errors_query",
+    "build_response_time_query",
+    "build_service_volume_query",
+    "build_user_activity_query",
+    "build_pattern_detection_query",
+    "build_tender_participant_query",
+    # Error handling
+    "RetryableError",
+    "NonRetryableError",
+    "ErrorPatternMatcher",
+    "ErrorAggregator",
+    "with_retry",
+    "retry_on_network_error",
+    "retry_on_db_error",
+    "retry_on_es_error",
+    "handle_lambda_error",
+    "categorize_retryable_error",
+    # CloudWatch metrics
+    "MetricsPublisher",
+    "MetricAggregator",
+    "StandardMetrics",
+    "TimedMetric",
+    "track_lambda_performance",
+    "create_service_dimensions",
+    "publish_health_metric",
+    # AWS Powertools integration
+    "get_powertools_logger",
+    "powertools_handler",
+    # Log processing
+    "extract_cloudwatch_logs_from_kinesis",
+    "derive_index_name",
+    "CloudWatchLogEvent",
+    "CloudWatchLogsData",
+    # Lambda context helpers
+    "get_lambda_environment_info",
+    # JWT authentication
+    "validate_jwt",
+    "require_auth",
+    "check_auth",
+    "get_jwt_public_key",
+    "JWTValidationError",
+    "AuthenticationError",
+]

nui_lambda_shared_utils/base_client.py

@@ -0,0 +1,323 @@
+"""
+Base client class providing common functionality for AWS service clients.
+"""
+
+import logging
+from typing import Optional, Dict, Any
+from abc import ABC, abstractmethod
+
+from .config import get_config
+from .secrets_helper import get_secret
+from .utils import resolve_config_value, validate_required_param, handle_client_errors
+
+log = logging.getLogger(__name__)
+
+
+class BaseClient(ABC):
+    """
+    Base class for AWS service clients providing standardized:
+    - Credential resolution and management
+    - Configuration integration
+    - Error handling patterns
+    - Logging context
+    """
+
+    def __init__(
+        self,
+        secret_name: Optional[str] = None,
+        config_key_prefix: Optional[str] = None,
+        credentials: Optional[Dict[str, Any]] = None,
+        **kwargs
+    ):
+        """
+        Initialize base client with standardized credential and config resolution.
+
+        Credential resolution precedence:
+        1. Explicit ``credentials`` dict — skip everything else
+        2. Environment variables (per-client patterns) — skip Secrets Manager
+        3. Secrets Manager (existing behavior) — unchanged
+
+        Args:
+            secret_name: Override secret name for credentials
+            config_key_prefix: Prefix for config keys (e.g., 'slack', 'es', 'db')
+            credentials: Direct credentials dict, bypasses Secrets Manager entirely
+            **kwargs: Additional client-specific parameters
+        """
+        self.config = get_config()
+        self.config_key_prefix = config_key_prefix or self._get_default_config_prefix()
+
+        # Resolve and store credentials
+        self.credentials = self._resolve_credentials(secret_name, credentials)
+
+        # Store additional configuration
+        self.client_config = kwargs
+        
+        # Initialize service-specific client
+        self._service_client = self._create_service_client()
+        
+        log.info(
+            f"Initialized {self.__class__.__name__}",
+            extra={
+                "client_type": self.__class__.__name__,
+                "config_prefix": self.config_key_prefix,
+                "has_credentials": bool(self.credentials)
+            }
+        )
+
+    @abstractmethod
+    def _get_default_config_prefix(self) -> str:
+        """Return the default configuration key prefix for this client type."""
+        pass
+
+    @abstractmethod
+    def _create_service_client(self) -> Any:
+        """Create and return the underlying service client (e.g., WebClient, Elasticsearch)."""
+        pass
+
+    @abstractmethod
+    def _get_default_secret_name(self) -> str:
+        """Return the default secret name for this client type."""
+        pass
+
+    def _resolve_credentials_from_env(self) -> Optional[Dict[str, Any]]:
+        """
+        Resolve credentials from environment variables.
+
+        Subclasses override this to check client-specific env vars
+        (e.g. SLACK_BOT_TOKEN, ES_PASSWORD).  Return a credentials dict
+        if the required env vars are present, or ``None`` to fall through
+        to Secrets Manager.
+
+        Returns:
+            Credentials dict or None
+        """
+        return None
+
+    def _resolve_credentials(
+        self,
+        secret_name: Optional[str],
+        explicit_credentials: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """
+        Resolve credentials using standardized precedence.
+
+        Precedence:
+        1. Explicit credentials dict (constructor param)
+        2. Environment variables (per-client ``_resolve_credentials_from_env``)
+        3. AWS Secrets Manager (via ``_fetch_credentials_from_sm``)
+
+        Args:
+            secret_name: Optional override for secret name
+            explicit_credentials: Direct credentials dict from constructor
+
+        Returns:
+            Dictionary containing resolved credentials
+        """
+        # 1. Explicit credentials dict — skip everything
+        if explicit_credentials is not None:
+            log.debug("Using explicitly provided credentials")
+            return explicit_credentials
+
+        # 2. Environment variables — skip Secrets Manager
+        env_credentials = self._resolve_credentials_from_env()
+        if env_credentials is not None:
+            log.debug("Using credentials from environment variables")
+            return env_credentials
+
+        # 3. Secrets Manager
+        return self._fetch_credentials_from_sm(secret_name)
+
+    def _resolve_secret_name(self, secret_name: Optional[str]) -> str:
+        """
+        Resolve the Secrets Manager secret name using standard precedence:
+        explicit param > env var > config default.
+
+        Args:
+            secret_name: Optional explicit override
+
+        Returns:
+            Resolved secret name string
+        """
+        resolved = resolve_config_value(
+            secret_name,
+            [
+                f"{self.config_key_prefix.upper()}_CREDENTIALS_SECRET",
+                f"{self.config_key_prefix.upper()}CREDENTIALS_SECRET"  # Alternative format
+            ],
+            getattr(self.config, f"{self.config_key_prefix}_credentials_secret", self._get_default_secret_name())
+        )
+        validate_required_param(resolved, "secret_name")
+        return resolved
+
+    def _fetch_credentials_from_sm(self, secret_name: Optional[str]) -> Dict[str, Any]:
+        """
+        Fetch credentials from AWS Secrets Manager.
+
+        Subclasses override this to customize the SM fetch (e.g. DB clients
+        use ``get_database_credentials`` for field normalization).
+
+        Args:
+            secret_name: Optional override for secret name
+
+        Returns:
+            Dictionary containing resolved credentials
+        """
+        resolved_secret_name = self._resolve_secret_name(secret_name)
+
+        try:
+            credentials = get_secret(resolved_secret_name)
+            log.debug(f"Retrieved credentials from secret: {resolved_secret_name}")
+            return credentials
+        except Exception as e:
+            log.error(
+                f"Failed to retrieve credentials from secret: {resolved_secret_name}",
+                extra={"secret_name": resolved_secret_name, "error": str(e)}
+            )
+            raise
+
+    def _get_config_value(self, key: str, env_vars: Optional[list] = None, default: Any = None) -> Any:
+        """
+        Get configuration value with standardized precedence.
+        
+        Args:
+            key: Configuration key (without prefix)
+            env_vars: Environment variable names to check
+            default: Default value
+            
+        Returns:
+            Resolved configuration value
+        """
+        # Get from client config (constructor kwargs) first
+        if key in self.client_config:
+            return self.client_config[key]
+        
+        # Build full config key
+        config_key = f"{self.config_key_prefix}_{key}"
+        config_value = getattr(self.config, config_key, default)
+        
+        # Use utility for full resolution
+        return resolve_config_value(
+            None,  # No explicit parameter (already checked client_config)
+            env_vars or [],
+            config_value
+        )
+
+    @handle_client_errors(reraise=True)
+    def _execute_with_error_handling(self, operation_name: str, operation_func, **log_context):
+        """
+        Execute an operation with standardized error handling and logging.
+        
+        Args:
+            operation_name: Name of the operation for logging
+            operation_func: Function to execute
+            **log_context: Additional logging context
+            
+        Returns:
+            Result of operation_func
+        """
+        context = {
+            "client_type": self.__class__.__name__,
+            "operation": operation_name,
+            **log_context
+        }
+        
+        log.debug(f"Executing {operation_name}", extra=context)
+        
+        try:
+            result = operation_func()
+            log.debug(f"Successfully completed {operation_name}", extra=context)
+            return result
+        except Exception as e:
+            context["error_type"] = type(e).__name__
+            context["error_message"] = str(e)
+            log.error(f"Failed to execute {operation_name}: {e}", extra=context)
+            raise
+
+    def get_client_info(self) -> Dict[str, Any]:
+        """
+        Get information about this client instance.
+        
+        Returns:
+            Dictionary with client information
+        """
+        return {
+            "client_type": self.__class__.__name__,
+            "config_prefix": self.config_key_prefix,
+            "has_credentials": bool(self.credentials),
+            "client_config": {k: v for k, v in self.client_config.items() if not k.startswith('_')},
+        }
+
+
+class ServiceHealthMixin:
+    """
+    Mixin providing common health check functionality for service clients.
+    """
+
+    def health_check(self) -> Dict[str, Any]:
+        """
+        Perform a basic health check for the service.
+        
+        Returns:
+            Dictionary with health status information
+        """
+        try:
+            self._perform_health_check()
+            return {
+                "status": "healthy",
+                "client_type": self.__class__.__name__,
+                "timestamp": log.time.time() if hasattr(log, 'time') else None
+            }
+        except Exception as e:
+            return {
+                "status": "unhealthy",
+                "client_type": self.__class__.__name__,
+                "error": str(e),
+                "error_type": type(e).__name__,
+                "timestamp": log.time.time() if hasattr(log, 'time') else None
+            }
+
+    @abstractmethod
+    def _perform_health_check(self):
+        """Perform service-specific health check. Should raise exception if unhealthy."""
+        pass
+
+
+class RetryableOperationMixin:
+    """
+    Mixin providing retry functionality for operations.
+    """
+
+    def execute_with_retry(
+        self,
+        operation_func,
+        operation_name: str,
+        max_attempts: int = 3,
+        **retry_kwargs
+    ):
+        """
+        Execute operation with retry logic.
+        
+        Args:
+            operation_func: Function to execute
+            operation_name: Name for logging
+            max_attempts: Maximum retry attempts
+            **retry_kwargs: Additional retry configuration
+            
+        Returns:
+            Result of operation_func
+        """
+        from .error_handler import with_retry
+        
+        # Apply retry decorator dynamically
+        retried_operation = with_retry(
+            max_attempts=max_attempts,
+            **retry_kwargs
+        )(operation_func)
+        
+        executor = getattr(self, "_execute_with_error_handling", None)
+        if executor is None:
+            raise AttributeError(
+                f"{self.__class__.__name__} must implement _execute_with_error_handling "
+                "to use RetryableOperationMixin"
+            )
+        return executor(operation_name, retried_operation)