nui-lambda-shared-utils 1.1.4__tar.gz → 1.1.5__tar.gz

{nui_lambda_shared_utils-1.1.4 → nui_lambda_shared_utils-1.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nui-lambda-shared-utils
-Version: 1.1.4
+Version: 1.1.5
 Summary: Enterprise-grade utilities for AWS Lambda functions with Slack, Elasticsearch, and monitoring integrations
 Home-page: https://github.com/nuimarkets/nui-lambda-shared-utils
 Author: NUI Markets

{nui_lambda_shared_utils-1.1.4 → nui_lambda_shared_utils-1.1.5}/docs/README.md

@@ -20,6 +20,7 @@ Component-specific guides for major features:
 - **[Lambda Context Helpers](guides/lambda-utilities.md)** - Environment info extraction for logging and metrics
 - **[Slack Integration](guides/slack-integration.md)** - Messaging, formatting, and file uploads
 - **[Elasticsearch Integration](guides/elasticsearch-integration.md)** - Search, bulk indexing, health checks
+- **[Log Processing](guides/log-processing.md)** - Kinesis log extraction and ES index naming
 - Database Connections (planned)
 - Error Handling Patterns (planned)
 - CloudWatch Metrics (planned)

nui_lambda_shared_utils-1.1.5/docs/guides/log-processing.md

@@ -0,0 +1,156 @@
+# Log Processing Guide
+
+This guide covers the log processing utilities for Lambda functions that stream
+CloudWatch logs to Elasticsearch via Kinesis.
+
+## Overview
+
+The `log_processors` module provides utilities for:
+
+- Extracting CloudWatch logs from Kinesis stream records
+- Deriving Elasticsearch index names from log metadata
+- Type definitions for CloudWatch log structures
+
+## Quick Start
+
+### Basic Usage
+
+```python
+from elasticsearch.helpers import streaming_bulk
+from nui_lambda_shared_utils.log_processors import (
+    extract_cloudwatch_logs_from_kinesis,
+    derive_index_name,
+)
+from datetime import datetime, timezone
+
+
+def process_log_events(log_group: str, log_stream: str, log_events: list):
+    """Process log events and yield ES documents."""
+    for event in log_events:
+        ts = datetime.fromtimestamp(event["timestamp"] / 1000.0, tz=timezone.utc)
+
+        yield {
+            "_index": derive_index_name(log_group, ts),
+            "_id": event["id"],
+            "_source": {
+                "message": event["message"],
+                "@timestamp": ts.isoformat(),
+                "log": {"group": log_group, "stream": log_stream},
+            }
+        }
+
+
+def handler(event, context):
+    """Lambda handler for Kinesis -> ES streaming."""
+    es = get_elasticsearch_client()
+
+    for ok, response in streaming_bulk(
+        client=es,
+        actions=extract_cloudwatch_logs_from_kinesis(
+            event["Records"],
+            process_fn=process_log_events
+        ),
+        chunk_size=100,
+        raise_on_error=True,
+    ):
+        if not ok:
+            logger.error(f"Document indexing failed: {response}")
+```
+
+### Error Handling
+
+Provide an `on_error` callback to handle failures without stopping the entire batch:
+
+```python
+def handle_processing_error(exception: Exception, record_data: dict):
+    """Log errors but continue processing."""
+    logger.error(f"Failed to process record: {exception}")
+    # Optionally send to dead letter queue, metrics, etc.
+
+
+for doc in extract_cloudwatch_logs_from_kinesis(
+    event["Records"],
+    process_fn=process_log_events,
+    on_error=handle_processing_error
+):
+    # Documents from successfully processed records
+    pass
+```
+
+### Custom Index Naming
+
+Override the default index naming pattern:
+
+```python
+from nui_lambda_shared_utils.log_processors import derive_index_name
+
+# Default: log-{service}-{YYYY}-m{MM}
+derive_index_name("/aws/lambda/orders", ts)
+# -> "log-orders-2025-m01"
+
+# Custom target
+derive_index_name("/aws/lambda/orders", ts, target_override="order-service")
+# -> "log-order-service-2025-m01"
+
+# Custom prefix and date format
+derive_index_name("/aws/lambda/orders", ts, prefix="logs", date_format="%Y-%m-%d")
+# -> "logs-orders-2025-01-15"
+```
+
+## Migration Guide
+
+### From inline Kinesis extraction
+
+Replace:
+
+```python
+# Before
+def extract_logs(records):
+    for row in records:
+        raw_data = row["kinesis"]["data"]
+        data = json.loads(
+            zlib.decompress(base64.b64decode(raw_data), 16 + zlib.MAX_WBITS).decode("utf-8")
+        )
+        if data["messageType"] == "CONTROL_MESSAGE":
+            continue
+        for item in process_log_events(...):
+            yield item
+```
+
+With:
+
+```python
+# After
+from nui_lambda_shared_utils.log_processors import extract_cloudwatch_logs_from_kinesis
+
+for doc in extract_cloudwatch_logs_from_kinesis(event["Records"], process_log_events):
+    yield doc
+```
+
+## API Reference
+
+### `extract_cloudwatch_logs_from_kinesis()`
+
+Extract CloudWatch logs from Kinesis stream records.
+
+**Parameters:**
+
+- `records`: List of Kinesis event records (`event["Records"]`)
+- `process_fn`: Callback to process log events
+- `on_error`: Optional error handler (if None, exceptions are raised)
+
+**Yields:** Dict documents ready for `streaming_bulk()`
+
+### `derive_index_name()`
+
+Derive Elasticsearch index name from log metadata.
+
+**Parameters:**
+
+- `log_group`: CloudWatch log group name
+- `timestamp`: Event timestamp for date suffix
+- `prefix`: Index name prefix (default: "log")
+- `date_format`: strftime format (default: "%Y-m%m")
+- `target_override`: Custom service name (optional)
+
+**Returns:** Index name string

{nui_lambda_shared_utils-1.1.4 → nui_lambda_shared_utils-1.1.5}/nui_lambda_shared_utils/__init__.py

@@ -121,6 +121,14 @@ except ImportError:
     get_powertools_logger = None  # type: ignore
     powertools_handler = None  # type: ignore
 
+# Log processing utilities (no external dependencies)
+from .log_processors import (
+    CloudWatchLogEvent,
+    CloudWatchLogsData,
+    derive_index_name,
+    extract_cloudwatch_logs_from_kinesis,
+)
+
 # Lambda context helpers (no external dependencies)
 from .lambda_helpers import get_lambda_environment_info
 
@@ -209,6 +217,11 @@ __all__ = [
     # AWS Powertools integration
     "get_powertools_logger",
     "powertools_handler",
+    # Log processing
+    "extract_cloudwatch_logs_from_kinesis",
+    "derive_index_name",
+    "CloudWatchLogEvent",
+    "CloudWatchLogsData",
     # Lambda context helpers
     "get_lambda_environment_info",
 ]

nui_lambda_shared_utils-1.1.5/nui_lambda_shared_utils/log_processors.py

@@ -0,0 +1,172 @@
+"""
+Utilities for extracting CloudWatch logs from Kinesis stream records.
+
+Provides standardized Kinesis log extraction, decompression, and index naming
+for Lambda functions that stream CloudWatch logs to Elasticsearch.
+"""
+
+import base64
+import json
+import logging
+import zlib
+from datetime import datetime
+from typing import Any, Callable, Dict, Iterator, List, Optional, TypedDict
+
+logger = logging.getLogger(__name__)
+
+
+class CloudWatchLogEvent(TypedDict):
+    """Single log event from CloudWatch."""
+
+    id: str
+    timestamp: int  # Unix timestamp in milliseconds
+    message: str
+
+
+class CloudWatchLogsData(TypedDict):
+    """Decompressed CloudWatch logs data structure."""
+
+    messageType: str  # "DATA_MESSAGE" or "CONTROL_MESSAGE"
+    owner: str
+    logGroup: str
+    logStream: str
+    subscriptionFilters: List[str]
+    logEvents: List[CloudWatchLogEvent]
+
+
+def extract_cloudwatch_logs_from_kinesis(
+    records: List[Dict[str, Any]],
+    process_fn: Callable[[str, str, List[Dict]], Iterator[Dict]],
+    on_error: Optional[Callable[[Exception, Dict], None]] = None,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Extract CloudWatch logs from Kinesis stream records.
+
+    Handles base64 decoding, gzip decompression, JSON parsing, and
+    CONTROL_MESSAGE filtering. Yields documents from the process_fn callback.
+
+    Args:
+        records: Kinesis event records (event["Records"])
+        process_fn: Callback to process log events. Signature:
+            process_fn(log_group: str, log_stream: str, log_events: List[Dict]) -> Iterator[Dict]
+            Should yield dicts with at minimum: {"_index": str, "_source": dict}
+        on_error: Optional error handler. If None, exceptions are raised.
+            Signature: on_error(exception: Exception, record_data: Dict) -> None
+
+    Yields:
+        Dict documents ready for Elasticsearch streaming_bulk()
+
+    Example:
+        from elasticsearch.helpers import streaming_bulk
+
+        def my_processor(log_group, log_stream, events):
+            for event in events:
+                yield {
+                    "_index": f"log-{log_group.split('/')[-1]}-2025-01",
+                    "_id": event["id"],
+                    "_source": {"message": event["message"], ...}
+                }
+
+        for ok, response in streaming_bulk(
+            client=es,
+            actions=extract_cloudwatch_logs_from_kinesis(
+                event["Records"],
+                process_fn=my_processor
+            )
+        ):
+            if not ok:
+                logger.error(f"Failed: {response}")
+    """
+    log_counts = []
+
+    for row in records:
+        try:
+            raw_data = row["kinesis"]["data"]
+        except (KeyError, TypeError) as e:
+            logger.exception("Kinesis record missing 'kinesis.data' key")
+            if on_error:
+                on_error(e, {"row": row})
+                continue
+            raise
+
+        try:
+            decompressed = zlib.decompress(
+                base64.b64decode(raw_data), 16 + zlib.MAX_WBITS
+            ).decode("utf-8")
+            data = json.loads(decompressed)
+        except Exception as e:
+            logger.exception("Failed to decode/decompress Kinesis record")
+            if on_error:
+                on_error(e, {"raw_data": raw_data[:100]})
+                continue
+            raise
+
+        try:
+            message_type = data["messageType"]
+            log_group = data["logGroup"]
+            log_stream = data["logStream"]
+            log_events = data["logEvents"]
+        except KeyError as e:
+            logger.exception("Malformed CloudWatch logs payload missing key: %s", e)
+            if on_error:
+                on_error(e, data)
+                continue
+            raise
+
+        if message_type == "CONTROL_MESSAGE":
+            logger.debug("Skipping CONTROL_MESSAGE")
+            continue
+
+        log_counts.append(len(log_events))
+
+        try:
+            yield from process_fn(log_group, log_stream, log_events)
+        except Exception as e:
+            logger.exception(f"Failed to process log events from {log_group}")
+            if on_error:
+                on_error(e, data)
+                continue
+            raise
+
+    logger.debug(
+        f"Processed {sum(log_counts)} log events from {len(records)} Kinesis records"
+    )
+
+
+def derive_index_name(
+    log_group: str,
+    timestamp: datetime,
+    prefix: str = "log",
+    date_format: str = "%Y-m%m",
+    target_override: Optional[str] = None,
+) -> str:
+    """
+    Derive Elasticsearch index name from log group and timestamp.
+
+    Default pattern: log-{service}-{YYYY}-m{MM}
+
+    Args:
+        log_group: CloudWatch log group name (e.g., "/aws/lambda/my-function")
+        timestamp: Event timestamp for date-based index suffix
+        prefix: Index name prefix (default: "log")
+        date_format: strftime format for date suffix (default: "%Y-m%m")
+        target_override: If provided, use this as service name instead of deriving from log_group
+
+    Returns:
+        Index name string (e.g., "log-my-function-2025-m01")
+
+    Example:
+        >>> derive_index_name("/aws/lambda/order-processor", datetime(2025, 1, 15))
+        'log-order-processor-2025-m01'
+
+        >>> derive_index_name("/ecs/my-service", datetime(2025, 1, 15), target_override="custom")
+        'log-custom-2025-m01'
+    """
+    if target_override:
+        service = target_override
+    else:
+        service = log_group.split("/")[-1]
+
+    date_suffix = timestamp.strftime(date_format)
+
+    return f"{prefix}-{service}-{date_suffix}".lower()

{nui_lambda_shared_utils-1.1.4 → nui_lambda_shared_utils-1.1.5}/nui_lambda_shared_utils/powertools_helpers.py

@@ -194,11 +194,15 @@ def powertools_handler(
 
         @functools.wraps(func)
         def wrapper(event: dict, context: Any) -> dict:
+            # Populate SlackClient account info from Lambda context ARN
+            if slack_client:
+                slack_client.set_handler_context(context)
+
             try:
                 # Apply logger context injection
                 # Note: inject_lambda_context is added dynamically to logging.Logger (line 95)
                 # and is native to Powertools Logger. Type checker can't verify this union.
-                handler_with_logging = logger.inject_lambda_context(func)  # type: ignore[attr-defined]
+                handler_with_logging = logger.inject_lambda_context(func)  # type: ignore[union-attr, attr-defined]
 
                 # Apply metrics if configured
                 if metrics:

{nui_lambda_shared_utils-1.1.4 → nui_lambda_shared_utils-1.1.5}/nui_lambda_shared_utils/slack_client.py

@@ -4,7 +4,7 @@ Refactored Slack client using BaseClient for DRY code patterns.
 
 import os
 import logging
-from typing import List, Dict, Optional
+from typing import Any, List, Dict, Optional
 from pathlib import Path
 from slack_sdk import WebClient
 from slack_sdk.errors import SlackApiError
@@ -165,6 +165,30 @@ class SlackClient(BaseClient, ServiceHealthMixin):
 
         return mappings
 
+    def set_handler_context(self, context: Any) -> None:
+        """
+        Extract AWS account ID from the Lambda handler context object.
+
+        Call this from your handler with the Lambda context to populate
+        account info without an STS API call. The powertools_handler
+        decorator calls this automatically.
+
+        Args:
+            context: Lambda context object (second arg to handler)
+        """
+        arn = getattr(context, "invoked_function_arn", None)
+        if not arn:
+            return
+
+        # ARN format: arn:aws:lambda:REGION:ACCOUNT_ID:function:NAME
+        parts = arn.split(":")
+        if len(parts) >= 5:
+            account_id = parts[4]
+            self._lambda_context["aws_account_id"] = account_id
+            self._lambda_context["aws_account_arn"] = arn
+            self._lambda_context["aws_account_name"] = self._account_names.get(account_id, "Unknown")
+            log.debug(f"Account ID from Lambda ARN: {account_id}")
+
     def _get_default_config_prefix(self) -> str:
         """Return configuration prefix for Slack."""
         return "slack"
@@ -204,23 +228,12 @@ class SlackClient(BaseClient, ServiceHealthMixin):
             "execution_env": os.environ.get("AWS_EXECUTION_ENV", "Unknown"),
         }
 
-        # Get AWS account info using utility
-        try:
-            sts_client = create_aws_client("sts")
-            account_info = sts_client.get_caller_identity()
-            context["aws_account_id"] = account_info.get("Account", "Unknown")
-            context["aws_account_arn"] = account_info.get("Arn", "Unknown")
-            context["aws_account_name"] = self._account_names.get(
-                context["aws_account_id"],
-                f"Unknown Account ({context['aws_account_id']})"
-            )
-        except Exception as e:
-            log.debug(f"Could not fetch AWS account info: {e}")
-            context.update({
-                "aws_account_id": "Unknown",
-                "aws_account_name": "Unknown", 
-                "aws_account_arn": "Unknown"
-            })
+        # Account info populated later by set_handler_context() from Lambda ARN
+        context.update({
+            "aws_account_id": "Unknown",
+            "aws_account_name": "Unknown",
+            "aws_account_arn": "Unknown",
+        })
 
         # Get deployment info
         context["deploy_time"] = self._get_deployment_age()
@@ -290,9 +303,10 @@ class SlackClient(BaseClient, ServiceHealthMixin):
         Returns:
             List of Slack blocks for Lambda context
         """
-        # Get account name from configured mappings or show as Unknown
+        # Build account display from pre-resolved name + ID
         account_id = self._lambda_context['aws_account_id']
-        simple_account = self._account_names.get(account_id, f"Unknown ({account_id})")
+        account_name = self._lambda_context['aws_account_name']
+        account_display = f"{account_name} ({account_id})" if account_id != "Unknown" else "Unknown"
 
         # Use custom service name if provided, otherwise use full function name
         display_name = self._service_name or self._lambda_context['function_name']
@@ -301,13 +315,14 @@ class SlackClient(BaseClient, ServiceHealthMixin):
         line1 = f"🤖 {display_name}"
         if event_type:
             line1 += f" • {event_type}"
-        line2 = f"{simple_account} ({account_id}) • {self._lambda_context['aws_region']} • 📋 Log: `{self._lambda_context['log_group']}`"
+        line2 = f"{account_display} • {self._lambda_context['aws_region']}"
+        line3 = f"📋 Log: `{self._lambda_context['log_group']}`"
 
         return [{
             "type": "context",
             "elements": [{
                 "type": "mrkdwn",
-                "text": f"{line1}\n{line2}"
+                "text": f"{line1}\n{line2}\n{line3}"
             }]
         }]
 
@@ -330,15 +345,14 @@ class SlackClient(BaseClient, ServiceHealthMixin):
             username = "Unknown"
 
         timestamp = datetime.now(timezone.utc).strftime("%H:%M UTC")
-        account_name = self._account_names.get(
-            self._lambda_context["aws_account_id"],
-            f"Unknown ({self._lambda_context['aws_account_id']})"
-        )
+        account_id = self._lambda_context["aws_account_id"]
+        account_name = self._lambda_context["aws_account_name"]
+        account_display = f"{account_name} ({account_id})" if account_id != "Unknown" else "Unknown"
 
         line1 = f"👤 `Local Testing` • {username}"
         if event_type:
             line1 += f" • {event_type}"
-        line2 = f"📍 {account_name} • {self._lambda_context['aws_region']} • {timestamp}"
+        line2 = f"📍 {account_display} • {self._lambda_context['aws_region']} • {timestamp}"
         line3 = "📋 Context: Manual/Development Testing"
 
         return [{
@@ -355,6 +369,7 @@ class SlackClient(BaseClient, ServiceHealthMixin):
         channel: str,
         text: str,
         blocks: Optional[List[Dict]] = None,
+        attachments: Optional[List[Dict]] = None,
         include_lambda_header: bool = True,
         event_type: Optional[str] = None
     ) -> bool:
@@ -365,6 +380,7 @@ class SlackClient(BaseClient, ServiceHealthMixin):
             channel: Channel ID
             text: Fallback text
             blocks: Rich formatted blocks
+            attachments: Legacy attachment objects (supports color sidebars)
             include_lambda_header: Whether to include context header
             event_type: Optional event type label for header (e.g., "Scheduled", "API", "SQS")
 
@@ -389,7 +405,8 @@ class SlackClient(BaseClient, ServiceHealthMixin):
             response = self._service_client.chat_postMessage(
                 channel=channel,
                 text=text,
-                blocks=blocks_with_header
+                blocks=blocks_with_header,
+                attachments=attachments
             )
 
             if response["ok"]:

{nui_lambda_shared_utils-1.1.4 → nui_lambda_shared_utils-1.1.5}/nui_lambda_shared_utils.egg-info/SOURCES.txt

@@ -21,6 +21,7 @@ docs/getting-started/quickstart.md
 docs/guides/cli-tools.md
 docs/guides/elasticsearch-integration.md
 docs/guides/lambda-utilities.md
+docs/guides/log-processing.md
 docs/guides/powertools-integration.md
 docs/guides/shared-types.md
 docs/guides/slack-integration.md
@@ -34,6 +35,7 @@ nui_lambda_shared_utils/error_handler.py
 nui_lambda_shared_utils/es_client.py
 nui_lambda_shared_utils/es_query_builder.py
 nui_lambda_shared_utils/lambda_helpers.py
+nui_lambda_shared_utils/log_processors.py
 nui_lambda_shared_utils/powertools_helpers.py
 nui_lambda_shared_utils/secrets_helper.py
 nui_lambda_shared_utils/slack_client.py
@@ -54,6 +56,7 @@ tests/test_error_handler.py
 tests/test_es_client.py
 tests/test_es_query_builder.py
 tests/test_lambda_helpers.py
+tests/test_log_processors.py
 tests/test_powertools_helpers.py
 tests/test_secrets_helper.py
 tests/test_slack_client.py

{nui_lambda_shared_utils-1.1.4 → nui_lambda_shared_utils-1.1.5}/tests/test_lambda_helpers.py

@@ -245,3 +245,9 @@ class TestExportedAPI:
         from nui_lambda_shared_utils.lambda_helpers import __all__
 
         assert "get_lambda_environment_info" in __all__
+
+    def test_importable_from_package_root(self):
+        """Test function is accessible from package root"""
+        from nui_lambda_shared_utils import get_lambda_environment_info
+
+        assert callable(get_lambda_environment_info)