nui-lambda-shared-utils 1.0.0__py3-none-any.whl

nui_lambda_shared_utils/__init__.py

@@ -0,0 +1,177 @@
+"""
+Enterprise-grade utilities for AWS Lambda functions with Slack, Elasticsearch, and monitoring integrations.
+"""
+
+# Configuration system
+from .config import (
+    Config,
+    get_config,
+    set_config,
+    configure,
+    get_es_host,
+    get_es_credentials_secret,
+    get_db_credentials_secret,
+    get_slack_credentials_secret,
+)
+
+# Core utilities
+from .secrets_helper import (
+    get_secret,
+    get_database_credentials,
+    get_elasticsearch_credentials,
+    get_slack_credentials,
+    get_api_key,
+    clear_cache,
+)
+
+# Optional imports - only fail if actually used
+try:
+    from .slack_client import SlackClient
+except ImportError:
+    SlackClient = None
+
+try:
+    from .es_client import ElasticsearchClient
+except ImportError:
+    ElasticsearchClient = None
+
+try:
+    from .db_client import DatabaseClient, get_pool_stats
+except ImportError:
+    DatabaseClient = None
+    get_pool_stats = None
+
+from .timezone import nz_time, format_nz_time
+
+# Slack formatting utilities (no external dependencies)
+from .slack_formatter import (
+    SlackBlockBuilder,
+    format_currency,
+    format_percentage,
+    format_number,
+    format_nz_time as format_nz_time_slack,
+    format_date_range,
+    format_daily_header,
+    format_weekly_header,
+    format_error_alert,
+    SERVICE_EMOJI,
+    SEVERITY_EMOJI,
+    STATUS_EMOJI,
+)
+
+# ES query builder - optional import
+try:
+    from .es_query_builder import (
+        ESQueryBuilder,
+        build_error_rate_query,
+        build_top_errors_query,
+        build_response_time_query,
+        build_service_volume_query,
+        build_user_activity_query,
+        build_pattern_detection_query,
+        build_tender_participant_query,
+    )
+except ImportError:
+    ESQueryBuilder = None
+    build_error_rate_query = None
+    build_top_errors_query = None
+    build_response_time_query = None
+    build_service_volume_query = None
+    build_user_activity_query = None
+    build_pattern_detection_query = None
+    build_tender_participant_query = None
+from .error_handler import (
+    RetryableError,
+    NonRetryableError,
+    ErrorPatternMatcher,
+    ErrorAggregator,
+    with_retry,
+    retry_on_network_error,
+    retry_on_db_error,
+    retry_on_es_error,
+    handle_lambda_error,
+    categorize_retryable_error,
+)
+from .cloudwatch_metrics import (
+    MetricsPublisher,
+    MetricAggregator,
+    StandardMetrics,
+    TimedMetric,
+    track_lambda_performance,
+    create_service_dimensions,
+    publish_health_metric,
+)
+
+
+# Slack setup utilities (for CLI usage) - optional import
+try:
+    from . import slack_setup
+except ImportError:
+    slack_setup = None
+
+__all__ = [
+    # Configuration system
+    "Config",
+    "get_config",
+    "set_config",
+    "configure",
+    "get_es_host",
+    "get_es_credentials_secret",
+    "get_db_credentials_secret",
+    "get_slack_credentials_secret",
+    # Core utilities
+    "get_secret",
+    "get_database_credentials",
+    "get_elasticsearch_credentials",
+    "get_slack_credentials",
+    "get_api_key",
+    "clear_cache",
+    "SlackClient",
+    "ElasticsearchClient",
+    "DatabaseClient",
+    "get_pool_stats",
+    "nz_time",
+    "format_nz_time",
+    "slack_setup",
+    # Slack formatting
+    "SlackBlockBuilder",
+    "format_currency",
+    "format_percentage",
+    "format_number",
+    "format_nz_time_slack",
+    "format_date_range",
+    "format_daily_header",
+    "format_weekly_header",
+    "format_error_alert",
+    "SERVICE_EMOJI",
+    "SEVERITY_EMOJI",
+    "STATUS_EMOJI",
+    # ES query building
+    "ESQueryBuilder",
+    "build_error_rate_query",
+    "build_top_errors_query",
+    "build_response_time_query",
+    "build_service_volume_query",
+    "build_user_activity_query",
+    "build_pattern_detection_query",
+    "build_tender_participant_query",
+    # Error handling
+    "RetryableError",
+    "NonRetryableError",
+    "ErrorPatternMatcher",
+    "ErrorAggregator",
+    "with_retry",
+    "retry_on_network_error",
+    "retry_on_db_error",
+    "retry_on_es_error",
+    "handle_lambda_error",
+    "categorize_retryable_error",
+    # CloudWatch metrics
+    "MetricsPublisher",
+    "MetricAggregator",
+    "StandardMetrics",
+    "TimedMetric",
+    "track_lambda_performance",
+    "create_service_dimensions",
+    "publish_health_metric",
+]

nui_lambda_shared_utils/cloudwatch_metrics.py

@@ -0,0 +1,367 @@
+"""
+CloudWatch metrics publishing utilities for Lambda services.
+Provides efficient batching and standardized metric publishing patterns.
+"""
+
+import os
+import time
+import logging
+from typing import Dict, List, Optional, Union, Any
+from datetime import datetime
+from collections import defaultdict
+import boto3
+from botocore.exceptions import ClientError
+
+log = logging.getLogger(__name__)
+
+
+class MetricsPublisher:
+    """
+    Efficient CloudWatch metrics publisher with batching support.
+
+    Example:
+        publisher = MetricsPublisher('Application')
+        publisher.put_metric('RecordsProcessed', 150, unit='Count')
+        publisher.put_metric('ResponseTime', 245.5, unit='Milliseconds')
+        publisher.flush()  # Send all metrics
+    """
+
+    def __init__(
+        self,
+        namespace: str,
+        dimensions: Optional[Dict[str, str]] = None,
+        auto_flush_size: int = 20,
+        region: Optional[str] = None,
+    ):
+        """
+        Initialize metrics publisher.
+
+        Args:
+            namespace: CloudWatch namespace for metrics
+            dimensions: Default dimensions to apply to all metrics
+            auto_flush_size: Automatically flush when batch reaches this size
+            region: AWS region (uses default if not specified)
+        """
+        self.namespace = namespace
+        self.default_dimensions = dimensions or {}
+        self.auto_flush_size = auto_flush_size
+        self.client = boto3.client("cloudwatch", region_name=region)
+        self.metric_buffer: List[Dict] = []
+
+    def put_metric(
+        self,
+        metric_name: str,
+        value: Union[int, float],
+        unit: str = "None",
+        timestamp: Optional[datetime] = None,
+        dimensions: Optional[Dict[str, str]] = None,
+        storage_resolution: int = 60,
+    ) -> None:
+        """
+        Add a metric to the buffer.
+
+        Args:
+            metric_name: Name of the metric
+            value: Metric value
+            unit: CloudWatch unit (Count, Milliseconds, Bytes, etc.)
+            timestamp: Metric timestamp (defaults to now)
+            dimensions: Additional dimensions for this metric
+            storage_resolution: 1 for high-resolution, 60 for standard
+        """
+        metric_data = {
+            "MetricName": metric_name,
+            "Value": float(value),
+            "Unit": unit,
+            "Timestamp": timestamp or datetime.utcnow(),
+            "StorageResolution": storage_resolution,
+        }
+
+        # Merge dimensions
+        all_dimensions = {**self.default_dimensions}
+        if dimensions:
+            all_dimensions.update(dimensions)
+
+        if all_dimensions:
+            metric_data["Dimensions"] = [{"Name": k, "Value": str(v)} for k, v in all_dimensions.items()]
+
+        self.metric_buffer.append(metric_data)
+
+        # Auto-flush if buffer is full
+        if len(self.metric_buffer) >= self.auto_flush_size:
+            self.flush()
+
+    def put_metric_with_statistics(
+        self,
+        metric_name: str,
+        values: List[Union[int, float]],
+        unit: str = "None",
+        timestamp: Optional[datetime] = None,
+        dimensions: Optional[Dict[str, str]] = None,
+    ) -> None:
+        """
+        Add a metric with statistical values.
+
+        Args:
+            metric_name: Name of the metric
+            values: List of values to calculate statistics from
+            unit: CloudWatch unit
+            timestamp: Metric timestamp
+            dimensions: Additional dimensions
+        """
+        if not values:
+            return
+
+        # Calculate statistics
+        sorted_values = sorted(values)
+        count = len(values)
+        sum_value = sum(values)
+        min_value = sorted_values[0]
+        max_value = sorted_values[-1]
+
+        metric_data = {
+            "MetricName": metric_name,
+            "StatisticValues": {"SampleCount": count, "Sum": sum_value, "Minimum": min_value, "Maximum": max_value},
+            "Unit": unit,
+            "Timestamp": timestamp or datetime.utcnow(),
+        }
+
+        # Merge dimensions
+        all_dimensions = {**self.default_dimensions}
+        if dimensions:
+            all_dimensions.update(dimensions)
+
+        if all_dimensions:
+            metric_data["Dimensions"] = [{"Name": k, "Value": str(v)} for k, v in all_dimensions.items()]
+
+        self.metric_buffer.append(metric_data)
+
+        if len(self.metric_buffer) >= self.auto_flush_size:
+            self.flush()
+
+    def flush(self) -> bool:
+        """
+        Send all buffered metrics to CloudWatch.
+
+        Returns:
+            bool: True if successful, False otherwise
+        """
+        if not self.metric_buffer:
+            return True
+
+        try:
+            # CloudWatch allows max 20 metrics per request
+            for i in range(0, len(self.metric_buffer), 20):
+                batch = self.metric_buffer[i : i + 20]
+                self.client.put_metric_data(Namespace=self.namespace, MetricData=batch)
+
+            log.info(f"Published {len(self.metric_buffer)} metrics to {self.namespace}")
+            self.metric_buffer = []
+            return True
+
+        except ClientError as e:
+            log.error(f"Failed to publish metrics: {e}")
+            return False
+
+    def __enter__(self):
+        """Context manager support."""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Ensure metrics are flushed on exit."""
+        self.flush()
+
+
+class MetricAggregator:
+    """
+    Aggregate metrics over time before publishing.
+    Useful for high-frequency metrics that need aggregation.
+    """
+
+    def __init__(self, publisher: MetricsPublisher):
+        self.publisher = publisher
+        self.aggregates: Dict[str, List[float]] = defaultdict(list)
+        self.counters: Dict[str, float] = defaultdict(float)
+
+    def add_value(self, metric_name: str, value: Union[int, float]) -> None:
+        """Add a value to be aggregated."""
+        self.aggregates[metric_name].append(float(value))
+
+    def increment_counter(self, metric_name: str, value: Union[int, float] = 1) -> None:
+        """Increment a counter metric."""
+        self.counters[metric_name] += value
+
+    def publish_aggregates(self, unit: str = "None", dimensions: Optional[Dict[str, str]] = None) -> None:
+        """Publish all aggregated metrics with statistics."""
+        # Publish aggregated values
+        for metric_name, values in self.aggregates.items():
+            if values:
+                self.publisher.put_metric_with_statistics(metric_name, values, unit=unit, dimensions=dimensions)
+
+        # Publish counters
+        for metric_name, value in self.counters.items():
+            self.publisher.put_metric(metric_name, value, unit="Count", dimensions=dimensions)
+
+        # Clear aggregates
+        self.aggregates.clear()
+        self.counters.clear()
+
+
+# Standard metric names for consistency across services
+class StandardMetrics:
+    """Standard metric names used across application services."""
+
+    # Service health metrics
+    SERVICE_HEALTH = "ServiceHealth"
+    ERROR_RATE = "ErrorRate"
+    RESPONSE_TIME = "ResponseTime"
+    REQUEST_COUNT = "RequestCount"
+
+    # Business metrics
+    RECORDS_CREATED = "RecordsCreated"
+    RECORDS_PROCESSED = "RecordsProcessed"
+    USERS_ACTIVE = "ActiveUsers"
+    REVENUE_PROCESSED = "RevenueProcessed"
+
+    # Lambda metrics
+    LAMBDA_DURATION = "LambdaDuration"
+    LAMBDA_ERRORS = "LambdaErrors"
+    LAMBDA_THROTTLES = "LambdaThrottles"
+    LAMBDA_COLD_STARTS = "LambdaColdStarts"
+
+    # Database metrics
+    DB_QUERY_TIME = "DatabaseQueryTime"
+    DB_CONNECTION_ERRORS = "DatabaseConnectionErrors"
+    DB_ACTIVE_CONNECTIONS = "DatabaseActiveConnections"
+
+    # Elasticsearch metrics
+    ES_QUERY_TIME = "ElasticsearchQueryTime"
+    ES_QUERY_ERRORS = "ElasticsearchQueryErrors"
+    ES_DOCUMENT_COUNT = "ElasticsearchDocumentCount"
+
+    # External API metrics
+    API_CALL_DURATION = "ExternalAPICallDuration"
+    API_CALL_ERRORS = "ExternalAPICallErrors"
+    API_RATE_LIMIT_HITS = "APIRateLimitHits"
+
+
+def track_lambda_performance(namespace: str = "Application"):
+    """
+    Decorator to automatically track Lambda function performance.
+
+    Example:
+        @track_lambda_performance()
+        def handler(event, context):
+            # Your Lambda logic
+            return response
+    """
+
+    def decorator(func):
+        def wrapper(event, context):
+            start_time = time.time()
+            cold_start = not hasattr(context, "is_warm")
+
+            publisher = MetricsPublisher(
+                namespace,
+                dimensions={"FunctionName": context.function_name, "Environment": os.environ.get("STAGE", "dev")},
+            )
+
+            try:
+                # Mark as warm for next invocation
+                context.is_warm = True
+
+                # Track cold start
+                if cold_start:
+                    publisher.put_metric(StandardMetrics.LAMBDA_COLD_STARTS, 1, unit="Count")
+
+                # Execute function
+                result = func(event, context)
+
+                # Track success metrics
+                duration = (time.time() - start_time) * 1000  # Convert to milliseconds
+                publisher.put_metric(StandardMetrics.LAMBDA_DURATION, duration, unit="Milliseconds")
+                publisher.put_metric(StandardMetrics.REQUEST_COUNT, 1, unit="Count")
+
+                return result
+
+            except Exception as e:
+                # Track error
+                publisher.put_metric(StandardMetrics.LAMBDA_ERRORS, 1, unit="Count")
+                raise
+
+            finally:
+                # Ensure metrics are published
+                publisher.flush()
+
+        return wrapper
+
+    return decorator
+
+
+def create_service_dimensions(service_name: str, environment: Optional[str] = None) -> Dict[str, str]:
+    """
+    Create standard dimensions for service metrics.
+
+    Args:
+        service_name: Name of the service
+        environment: Environment (defaults to STAGE env var)
+
+    Returns:
+        Dict of dimensions
+    """
+    dimensions = {"ServiceName": service_name, "Environment": environment or os.environ.get("STAGE", "dev")}
+
+    # Add region if available
+    region = os.environ.get("AWS_REGION", os.environ.get("AWS_DEFAULT_REGION"))
+    if region:
+        dimensions["Region"] = region
+
+    return dimensions
+
+
+def publish_health_metric(service_name: str, is_healthy: bool, namespace: str = "Application") -> None:
+    """
+    Publish a simple health metric for a service.
+
+    Args:
+        service_name: Name of the service
+        is_healthy: Whether the service is healthy
+        namespace: CloudWatch namespace
+    """
+    publisher = MetricsPublisher(namespace, dimensions=create_service_dimensions(service_name))
+
+    publisher.put_metric(StandardMetrics.SERVICE_HEALTH, 1 if is_healthy else 0, unit="None")
+
+    publisher.flush()
+
+
+class TimedMetric:
+    """
+    Context manager for timing operations and publishing metrics.
+
+    Example:
+        publisher = MetricsPublisher('MyApp')
+        with TimedMetric(publisher, 'DatabaseQuery', unit='Milliseconds'):
+            # Perform database query
+            results = db.query("SELECT * FROM records")
+    """
+
+    def __init__(
+        self,
+        publisher: MetricsPublisher,
+        metric_name: str,
+        unit: str = "Milliseconds",
+        dimensions: Optional[Dict[str, str]] = None,
+    ):
+        self.publisher = publisher
+        self.metric_name = metric_name
+        self.unit = unit
+        self.dimensions = dimensions
+        self.start_time = None
+
+    def __enter__(self):
+        self.start_time = time.time()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        duration = (time.time() - self.start_time) * 1000  # Convert to milliseconds
+        self.publisher.put_metric(self.metric_name, duration, unit=self.unit, dimensions=self.dimensions)

nui_lambda_shared_utils/config.py

@@ -0,0 +1,127 @@
+"""
+Configuration system for AWS Lambda shared utilities.
+
+This module provides configurable defaults and environment-based overrides
+to make the library suitable for different deployment environments.
+"""
+
+import os
+from typing import Dict, Optional, Any
+
+
+class Config:
+    """Configuration class for AWS Lambda shared utilities with environment-based overrides."""
+
+    def __init__(
+        self,
+        # Elasticsearch configuration
+        es_host: Optional[str] = None,
+        es_credentials_secret: Optional[str] = None,
+        # Database configuration
+        db_credentials_secret: Optional[str] = None,
+        # Slack configuration
+        slack_credentials_secret: Optional[str] = None,
+        # AWS configuration
+        aws_region: Optional[str] = None,
+    ):
+        """
+        Initialize configuration with optional overrides.
+
+        Args:
+            es_host: Elasticsearch host (default: localhost:9200)
+            es_credentials_secret: AWS secret name for ES credentials
+            db_credentials_secret: AWS secret name for database credentials
+            slack_credentials_secret: AWS secret name for Slack credentials
+            aws_region: AWS region for secrets/services
+        """
+        # Elasticsearch settings
+        self.es_host = es_host or os.environ.get("ES_HOST") or os.environ.get("ELASTICSEARCH_HOST") or "localhost:9200"
+
+        self.es_credentials_secret = (
+            es_credentials_secret
+            or os.environ.get("ES_CREDENTIALS_SECRET")
+            or os.environ.get("ELASTICSEARCH_CREDENTIALS_SECRET")
+            or "elasticsearch-credentials"
+        )
+
+        # Database settings
+        self.db_credentials_secret = (
+            db_credentials_secret
+            or os.environ.get("DB_CREDENTIALS_SECRET")
+            or os.environ.get("DATABASE_CREDENTIALS_SECRET")
+            or "database-credentials"
+        )
+
+        # Slack settings
+        self.slack_credentials_secret = (
+            slack_credentials_secret or os.environ.get("SLACK_CREDENTIALS_SECRET") or "slack-credentials"
+        )
+
+        # AWS settings
+        self.aws_region = (
+            aws_region or os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or "us-east-1"
+        )
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return configuration as dictionary for debugging/logging."""
+        return {
+            "es_host": self.es_host,
+            "es_credentials_secret": self.es_credentials_secret,
+            "db_credentials_secret": self.db_credentials_secret,
+            "slack_credentials_secret": self.slack_credentials_secret,
+            "aws_region": self.aws_region,
+        }
+
+
+# Global default configuration instance
+_default_config = None
+
+
+def get_config() -> Config:
+    """Get the current global configuration instance."""
+    global _default_config
+    if _default_config is None:
+        _default_config = Config()
+    return _default_config
+
+
+def set_config(config: Config) -> None:
+    """Set a new global configuration instance."""
+    global _default_config
+    _default_config = config
+
+
+def configure(**kwargs) -> Config:
+    """
+    Configure the global configuration with keyword arguments.
+
+    This is a convenience function equivalent to:
+    set_config(Config(**kwargs))
+
+    Returns:
+        The new configuration instance
+    """
+    config = Config(**kwargs)
+    set_config(config)
+    return config
+
+
+# Legacy compatibility - environment variable checking functions
+def get_es_host() -> str:
+    """Get Elasticsearch host from configuration (legacy compatibility)."""
+    return get_config().es_host
+
+
+def get_es_credentials_secret() -> str:
+    """Get Elasticsearch credentials secret name (legacy compatibility)."""
+    return get_config().es_credentials_secret
+
+
+def get_db_credentials_secret() -> str:
+    """Get database credentials secret name (legacy compatibility)."""
+    return get_config().db_credentials_secret
+
+
+def get_slack_credentials_secret() -> str:
+    """Get Slack credentials secret name (legacy compatibility)."""
+    return get_config().slack_credentials_secret