nui-python-shared-utils 1.3.0__py3-none-any.whl

nui_lambda_shared_utils/log_processors.py

@@ -0,0 +1,172 @@
+"""
+Utilities for extracting CloudWatch logs from Kinesis stream records.
+
+Provides standardized Kinesis log extraction, decompression, and index naming
+for Lambda functions that stream CloudWatch logs to Elasticsearch.
+"""
+
+import base64
+import json
+import logging
+import zlib
+from datetime import datetime
+from typing import Any, Callable, Dict, Iterator, List, Optional, TypedDict
+
+logger = logging.getLogger(__name__)
+
+
+class CloudWatchLogEvent(TypedDict):
+    """Single log event from CloudWatch."""
+
+    id: str
+    timestamp: int  # Unix timestamp in milliseconds
+    message: str
+
+
+class CloudWatchLogsData(TypedDict):
+    """Decompressed CloudWatch logs data structure."""
+
+    messageType: str  # "DATA_MESSAGE" or "CONTROL_MESSAGE"
+    owner: str
+    logGroup: str
+    logStream: str
+    subscriptionFilters: List[str]
+    logEvents: List[CloudWatchLogEvent]
+
+
+def extract_cloudwatch_logs_from_kinesis(
+    records: List[Dict[str, Any]],
+    process_fn: Callable[[str, str, List[Dict]], Iterator[Dict]],
+    on_error: Optional[Callable[[Exception, Dict], None]] = None,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Extract CloudWatch logs from Kinesis stream records.
+
+    Handles base64 decoding, gzip decompression, JSON parsing, and
+    CONTROL_MESSAGE filtering. Yields documents from the process_fn callback.
+
+    Args:
+        records: Kinesis event records (event["Records"])
+        process_fn: Callback to process log events. Signature:
+            process_fn(log_group: str, log_stream: str, log_events: List[Dict]) -> Iterator[Dict]
+            Should yield dicts with at minimum: {"_index": str, "_source": dict}
+        on_error: Optional error handler. If None, exceptions are raised.
+            Signature: on_error(exception: Exception, record_data: Dict) -> None
+
+    Yields:
+        Dict documents ready for Elasticsearch streaming_bulk()
+
+    Example:
+        from elasticsearch.helpers import streaming_bulk
+
+        def my_processor(log_group, log_stream, events):
+            for event in events:
+                yield {
+                    "_index": f"log-{log_group.split('/')[-1]}-2025-01",
+                    "_id": event["id"],
+                    "_source": {"message": event["message"], ...}
+                }
+
+        for ok, response in streaming_bulk(
+            client=es,
+            actions=extract_cloudwatch_logs_from_kinesis(
+                event["Records"],
+                process_fn=my_processor
+            )
+        ):
+            if not ok:
+                logger.error(f"Failed: {response}")
+    """
+    log_counts = []
+
+    for row in records:
+        try:
+            raw_data = row["kinesis"]["data"]
+        except (KeyError, TypeError) as e:
+            logger.exception("Kinesis record missing 'kinesis.data' key")
+            if on_error:
+                on_error(e, {"row": row})
+                continue
+            raise
+
+        try:
+            decompressed = zlib.decompress(
+                base64.b64decode(raw_data), 16 + zlib.MAX_WBITS
+            ).decode("utf-8")
+            data = json.loads(decompressed)
+        except Exception as e:
+            logger.exception("Failed to decode/decompress Kinesis record")
+            if on_error:
+                on_error(e, {"raw_data": raw_data[:100]})
+                continue
+            raise
+
+        try:
+            message_type = data["messageType"]
+            log_group = data["logGroup"]
+            log_stream = data["logStream"]
+            log_events = data["logEvents"]
+        except KeyError as e:
+            logger.exception("Malformed CloudWatch logs payload missing key: %s", e)
+            if on_error:
+                on_error(e, data)
+                continue
+            raise
+
+        if message_type == "CONTROL_MESSAGE":
+            logger.debug("Skipping CONTROL_MESSAGE")
+            continue
+
+        log_counts.append(len(log_events))
+
+        try:
+            yield from process_fn(log_group, log_stream, log_events)
+        except Exception as e:
+            logger.exception(f"Failed to process log events from {log_group}")
+            if on_error:
+                on_error(e, data)
+                continue
+            raise
+
+    logger.debug(
+        f"Processed {sum(log_counts)} log events from {len(records)} Kinesis records"
+    )
+
+
+def derive_index_name(
+    log_group: str,
+    timestamp: datetime,
+    prefix: str = "log",
+    date_format: str = "%Y-m%m",
+    target_override: Optional[str] = None,
+) -> str:
+    """
+    Derive Elasticsearch index name from log group and timestamp.
+
+    Default pattern: log-{service}-{YYYY}-m{MM}
+
+    Args:
+        log_group: CloudWatch log group name (e.g., "/aws/lambda/my-function")
+        timestamp: Event timestamp for date-based index suffix
+        prefix: Index name prefix (default: "log")
+        date_format: strftime format for date suffix (default: "%Y-m%m")
+        target_override: If provided, use this as service name instead of deriving from log_group
+
+    Returns:
+        Index name string (e.g., "log-my-function-2025-m01")
+
+    Example:
+        >>> derive_index_name("/aws/lambda/order-processor", datetime(2025, 1, 15))
+        'log-order-processor-2025-m01'
+
+        >>> derive_index_name("/ecs/my-service", datetime(2025, 1, 15), target_override="custom")
+        'log-custom-2025-m01'
+    """
+    if target_override:
+        service = target_override
+    else:
+        service = log_group.split("/")[-1]
+
+    date_suffix = timestamp.strftime(date_format)
+
+    return f"{prefix}-{service}-{date_suffix}".lower()

nui_lambda_shared_utils/powertools_helpers.py

@@ -0,0 +1,263 @@
+"""
+AWS Powertools integration utilities for Lambda functions.
+
+Provides standardized logging, metrics, and error handling patterns using AWS Lambda Powertools.
+"""
+
+import functools
+import logging
+import os
+from typing import Any, Callable, Dict, Optional, Union
+
+# Optional imports with graceful degradation
+try:
+    from aws_lambda_powertools import Logger, Metrics
+
+    POWERTOOLS_AVAILABLE = True
+except ImportError:
+    POWERTOOLS_AVAILABLE = False
+    Logger = None  # type: ignore
+    Metrics = None  # type: ignore
+
+try:
+    import coloredlogs
+
+    COLOREDLOGS_AVAILABLE = True
+except ImportError:
+    COLOREDLOGS_AVAILABLE = False
+
+try:
+    from .slack_client import SlackClient
+
+    SLACK_CLIENT_AVAILABLE = True
+except ImportError:
+    SLACK_CLIENT_AVAILABLE = False
+    SlackClient = None  # type: ignore
+
+from .lambda_helpers import get_lambda_environment_info
+
+
+__all__ = ["get_powertools_logger", "powertools_handler"]
+
+
+def get_powertools_logger(
+    service_name: str,
+    level: str = "INFO",
+    local_dev_colors: bool = True,
+) -> Union[Logger, logging.Logger]:
+    """
+    Create AWS Powertools Logger with Elasticsearch-compatible formatting.
+
+    Automatically detects Lambda environment and configures appropriate logging:
+    - Lambda environment: AWS Powertools Logger with JSON structured logging
+    - Local environment: Standard Python logger with coloredlogs (if available)
+
+    The logger uses Elasticsearch-compatible timestamp format (%Y-%m-%dT%H:%M:%SZ)
+    and enforces UTC timezone for consistency with log aggregation systems.
+
+    Args:
+        service_name: Service identifier (e.g., "nui-tender-analyser", "connect-email-ingest")
+        level: Log level (DEBUG, INFO, WARNING, ERROR, CRITICAL). Default: INFO
+        local_dev_colors: Enable coloredlogs for local development. Default: True
+
+    Returns:
+        Logger instance with inject_lambda_context decorator method.
+        - In Lambda: AWS Powertools Logger with JSON formatting
+        - Locally: Python Logger with mock inject_lambda_context method
+
+    Raises:
+        ImportError: If aws-lambda-powertools is not installed when running in Lambda environment
+
+    Example:
+        >>> logger = get_powertools_logger("my-service", level="INFO")
+        >>> @logger.inject_lambda_context
+        ... def handler(event, context):
+        ...     logger.info("Processing event", extra={"event_type": event.get("type")})
+        ...     return {"statusCode": 200}
+    """
+    # Detect Lambda environment
+    env_info = get_lambda_environment_info()
+    is_sam_local = os.getenv("AWS_SAM_LOCAL") is not None
+
+    # Local development environment (or SAM local for dev-friendly logging)
+    if env_info["is_local"] or is_sam_local:
+        logging.captureWarnings(True)
+
+        # Use coloredlogs for local development if available and enabled
+        if COLOREDLOGS_AVAILABLE and local_dev_colors:
+            # Clear root logger handlers before coloredlogs to avoid duplicates
+            logging.getLogger().handlers = []
+            coloredlogs.install(level=level, isatty=True)
+
+        # Create standard Python logger
+        logger = logging.getLogger(service_name)
+        logger.setLevel(level)
+
+        # Add mock inject_lambda_context decorator for local compatibility
+        # Must handle both @logger.inject_lambda_context and @logger.inject_lambda_context(log_event=False)
+        def _mock_inject_lambda_context(func=None, **_kwargs):
+            if func is not None:
+                return func
+            return lambda f: f
+        logger.inject_lambda_context = _mock_inject_lambda_context  # type: ignore
+
+        return logger
+
+    # Lambda environment - use AWS Powertools
+    if not POWERTOOLS_AVAILABLE:
+        raise ImportError(
+            "aws-lambda-powertools is required for Lambda environment. "
+            "Install with: pip install nui-python-shared-utils[powertools]"
+        )
+
+    # Create Powertools Logger with ES-compatible timestamp format
+    # Powertools default: '2025-01-18 04:39:27,788+0000'
+    # Elasticsearch expects: '2025-01-18T04:39:27Z' (ISO 8601)
+    # Note: %f (microseconds) is not supported by time.strftime() which logging uses internally
+    powertools_logger = Logger(
+        service=service_name,
+        level=level,
+        sampling_rate=1,
+        datefmt="%Y-%m-%dT%H:%M:%SZ",
+        utc=True,
+    )
+
+    return powertools_logger
+
+
+def powertools_handler(
+    service_name: str,
+    metrics_namespace: Optional[str] = None,
+    slack_alert_channel: Optional[str] = None,
+    slack_account_names: Optional[Dict[str, str]] = None,
+    slack_account_names_config: Optional[str] = None,
+):
+    """
+    Decorator for Lambda handlers with logging, metrics, and error handling.
+
+    Combines AWS Powertools Logger and Metrics decorators with automatic exception
+    handling and optional Slack alerting. Provides consistent error responses and
+    structured logging for Lambda functions.
+
+    Features:
+    - Automatic logger.inject_lambda_context integration
+    - Optional metrics.log_metrics integration (if metrics_namespace provided)
+    - Structured exception logging with traceback
+    - Optional Slack alerts on failures (if slack_alert_channel provided)
+    - Graceful degradation if Slack client unavailable
+    - Proper Lambda error response formatting
+
+    Args:
+        service_name: Service identifier for logging and metrics dimensions
+        metrics_namespace: CloudWatch namespace for metrics (e.g., "NUI/TenderAnalyser").
+                          If None, metrics publishing is disabled.
+        slack_alert_channel: Slack channel for error alerts (e.g., "#alerts", "#errors").
+                            If None, Slack alerting is disabled.
+        slack_account_names: Dict mapping AWS account IDs to display names
+        slack_account_names_config: Path to YAML file with account_names mapping.
+            The file must be bundled in the Lambda deployment package/zip.
+            Values loaded from this file are used as defaults, but any keys
+            provided in ``slack_account_names`` take precedence and override
+            the YAML-loaded values.
+
+    Returns:
+        Decorator function for Lambda handlers
+
+    Example:
+        >>> @powertools_handler(
+        ...     service_name="my-lambda",
+        ...     metrics_namespace="MyApp/Lambda",
+        ...     slack_alert_channel="#errors"
+        ... )
+        ... def handler(event, context):
+        ...     logger.info("Processing event")
+        ...     return {"statusCode": 200, "body": "Success"}
+
+    Example (minimal):
+        >>> @powertools_handler(service_name="simple-lambda")
+        ... def handler(event, context):
+        ...     return {"statusCode": 200}
+
+    Note:
+        The decorated handler must return a dict with statusCode and optional body.
+        On exception, returns: {"statusCode": 500, "body": "Internal Server Error"}
+    """
+
+    def decorator(func: Callable) -> Callable:
+        # Create logger
+        logger = get_powertools_logger(service_name)
+
+        # Create metrics publisher if namespace provided
+        metrics = None
+        if metrics_namespace and POWERTOOLS_AVAILABLE:
+            metrics = Metrics(namespace=metrics_namespace, service=service_name)
+
+        # Create Slack client if channel provided
+        slack_client = None
+        if slack_alert_channel and SLACK_CLIENT_AVAILABLE:
+            try:
+                slack_client = SlackClient(
+                    account_names=slack_account_names,
+                    account_names_config=slack_account_names_config,
+                )
+            except Exception as e:
+                logger.warning("Failed to initialize Slack client: %s", e)
+
+        @functools.wraps(func)
+        def wrapper(event: dict, context: Any) -> dict:
+            # Populate SlackClient account info from Lambda context ARN
+            if slack_client:
+                slack_client.set_handler_context(context)
+
+            try:
+                # Apply logger context injection
+                # Note: inject_lambda_context is added dynamically to logging.Logger (line 95)
+                # and is native to Powertools Logger. Type checker can't verify this union.
+                handler_with_logging = logger.inject_lambda_context(func)  # type: ignore[union-attr, attr-defined]
+
+                # Apply metrics if configured
+                if metrics:
+                    handler_with_metrics = metrics.log_metrics(handler_with_logging)
+                    result = handler_with_metrics(event, context)
+                else:
+                    result = handler_with_logging(event, context)
+
+                return result
+
+            except Exception as e:
+                # Log exception with full context
+                logger.exception(
+                    "Lambda handler failed: %s",
+                    str(e),
+                    extra={
+                        "error_type": type(e).__name__,
+                        "error_message": str(e),
+                        "service": service_name,
+                    },
+                )
+
+                # Send Slack alert if configured
+                if slack_client and slack_alert_channel:
+                    try:
+                        error_message = f"*Lambda Error: {service_name}*\n\n"
+                        error_message += f"Error: `{type(e).__name__}: {str(e)}`\n"
+                        error_message += (
+                            f"Function: `{context.function_name if hasattr(context, 'function_name') else 'unknown'}`"
+                        )
+
+                        slack_client.send_message(
+                            channel=slack_alert_channel,
+                            text=error_message,
+                        )
+                    except Exception as slack_error:
+                        logger.warning("Failed to send Slack alert: %s", slack_error)
+
+                # Return proper Lambda error response
+                return {
+                    "statusCode": 500,
+                    "body": "Internal Server Error",
+                }
+
+        return wrapper
+
+    return decorator

nui_lambda_shared_utils/secrets_helper.py

@@ -0,0 +1,187 @@
+"""
+AWS Secrets Manager helper for retrieving credentials.
+Shared across all AWS Lambda functions.
+"""
+
+import os
+import json
+import logging
+from typing import Dict, Optional
+import boto3
+from botocore.exceptions import ClientError
+
+from .config import get_config
+
+log = logging.getLogger(__name__)
+
+# Cache for secrets to avoid repeated API calls
+_secrets_cache = {}
+
+
+def get_secret(secret_name: str) -> Dict:
+    """
+    Retrieve secret from AWS Secrets Manager.
+
+    Args:
+        secret_name: Name of the secret in Secrets Manager
+
+    Returns:
+        Dict containing the secret values
+
+    Raises:
+        Exception if secret cannot be retrieved
+    """
+    # Check cache first
+    if secret_name in _secrets_cache:
+        return _secrets_cache[secret_name]
+
+    # Create a Secrets Manager client
+    session = boto3.session.Session()
+    client = session.client(service_name="secretsmanager", region_name=session.region_name or "ap-southeast-2")
+
+    try:
+        response = client.get_secret_value(SecretId=secret_name)
+
+        # Secrets Manager stores either a string or binary
+        if "SecretString" in response:
+            secret = json.loads(response["SecretString"])
+        else:
+            # Binary secret (not typically used for credentials)
+            secret = json.loads(response["SecretBinary"].decode("utf-8"))
+
+        # Cache the secret
+        _secrets_cache[secret_name] = secret
+
+        log.info(f"Successfully retrieved secret: {secret_name}")
+        return secret
+
+    except ClientError as e:
+        error_code = e.response["Error"]["Code"]
+
+        if error_code == "DecryptionFailureException":
+            log.error(f"Cannot decrypt secret {secret_name}: {e}")
+            raise Exception(f"Cannot decrypt secret {secret_name}")
+        elif error_code == "InternalServiceErrorException":
+            log.error(f"Internal service error retrieving {secret_name}: {e}")
+            raise Exception(f"Internal service error retrieving {secret_name}")
+        elif error_code == "InvalidParameterException":
+            log.error(f"Invalid parameter for {secret_name}: {e}")
+            raise Exception(f"Invalid parameter for {secret_name}")
+        elif error_code == "InvalidRequestException":
+            log.error(f"Invalid request for {secret_name}: {e}")
+            raise Exception(f"Invalid request for {secret_name}")
+        elif error_code == "ResourceNotFoundException":
+            log.error(f"Secret {secret_name} not found: {e}")
+            raise Exception(f"Secret {secret_name} not found")
+        else:
+            log.error(f"Unknown error retrieving {secret_name}: {e}")
+            raise Exception(f"Error retrieving secret {secret_name}: {error_code}")
+    except Exception as e:
+        log.error(f"Unexpected error retrieving {secret_name}: {e}")
+        raise Exception(f"Unexpected error retrieving secret {secret_name}: {str(e)}")
+
+
+def get_database_credentials(secret_name: Optional[str] = None) -> Dict:
+    """
+    Get database credentials with standardized field names.
+
+    Args:
+        secret_name: Override default from configuration or environment
+
+    Returns:
+        Dict with host, port, username, password, database
+    """
+    config = get_config()
+    secret = secret_name or os.environ.get("DB_CREDENTIALS_SECRET") or config.db_credentials_secret
+    if not secret:
+        raise ValueError("No database secret name provided")
+
+    creds = get_secret(secret)
+
+    # Normalize field names
+    return {
+        "host": creds.get("host", creds.get("endpoint", creds.get("hostname"))),
+        "port": int(creds.get("port", 3306)),
+        "username": creds.get("username", creds.get("user")),
+        "password": creds.get("password"),
+        "database": creds.get("database", creds.get("dbname", "app")),
+    }
+
+
+def get_elasticsearch_credentials(secret_name: Optional[str] = None) -> Dict:
+    """
+    Get Elasticsearch credentials.
+
+    Args:
+        secret_name: Override default from configuration or environment
+
+    Returns:
+        Dict with host, username, password
+    """
+    config = get_config()
+    secret = secret_name or os.environ.get("ES_CREDENTIALS_SECRET") or config.es_credentials_secret
+    if not secret:
+        raise ValueError("No Elasticsearch secret name provided")
+
+    creds = get_secret(secret)
+
+    # Use configuration system for host defaults instead of hardcoded value
+    host = os.environ.get("ES_HOST") or creds.get("host") or config.es_host
+    # Ensure port is included if not already present
+    if ":" not in host and not host.startswith("http"):
+        host = f"{host}:9200"
+
+    return {
+        "host": host,
+        "username": creds.get("username", "elastic"),
+        "password": creds.get("password"),
+    }
+
+
+def get_slack_credentials(secret_name: Optional[str] = None) -> Dict:
+    """
+    Get Slack bot credentials.
+
+    Args:
+        secret_name: Override default from configuration or environment
+
+    Returns:
+        Dict with bot_token and optional webhook_url
+    """
+    config = get_config()
+    secret = secret_name or os.environ.get("SLACK_CREDENTIALS_SECRET") or config.slack_credentials_secret
+    if not secret:
+        raise ValueError("No Slack secret name provided")
+
+    creds = get_secret(secret)
+
+    return {
+        "bot_token": creds.get("bot_token", creds.get("token")),
+        "webhook_url": creds.get("webhook_url"),  # Optional
+    }
+
+
+def get_api_key(secret_name: str, key_field: str = "api_key") -> str:
+    """
+    Get a simple API key from secrets.
+
+    Args:
+        secret_name: Name of the secret
+        key_field: Field name containing the key (default: 'api_key')
+
+    Returns:
+        The API key string
+    """
+    secret = get_secret(secret_name)
+
+    if key_field not in secret:
+        raise KeyError(f"Field '{key_field}' not found in secret {secret_name}")
+
+    return secret[key_field]
+
+
+def clear_cache() -> None:
+    """Clear the secrets cache. Useful for long-running Lambdas."""
+    global _secrets_cache
+    _secrets_cache.clear()
+    log.info("Cleared secrets cache")