nui-python-shared-utils 1.4.1__py3-none-any.whl → 1.6.0__py3-none-any.whl

{nui_python_shared_utils-1.4.1.dist-info → nui_python_shared_utils-1.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nui-python-shared-utils
-Version: 1.4.1
+Version: 1.6.0
 Summary: Shared Python utilities for AWS Lambda, CLI tools, and agents with Slack, Elasticsearch, and monitoring integrations
 Home-page: https://github.com/nuimarkets/nui-python-shared-utils
 Author: NUI Markets

{nui_python_shared_utils-1.4.1.dist-info → nui_python_shared_utils-1.6.0.dist-info}/RECORD

@@ -21,8 +21,8 @@ nui_lambda_shared_utils/slack_setup/__init__.py,sha256=q78NzSznsSd3ri5OUohWxQt7r
 nui_lambda_shared_utils/slack_setup/channel_creator.py,sha256=xaeqzyEAqVqYj2EhN69UvcyZQ6LdFKYODPPCwxHInLA,172
 nui_lambda_shared_utils/slack_setup/channel_definitions.py,sha256=3TkdLzWvEMBMDy5jqOPMUGRgKQ3z9TN58D43V7O7LpE,180
 nui_lambda_shared_utils/slack_setup/setup_helpers.py,sha256=ex_RiUfiZqH9qGLQwmedezMbfp35uI-VQPevducfbBk,168
-nui_python_shared_utils-1.4.1.dist-info/licenses/LICENSE,sha256=vGe2mC5yLUb8toYlY3T36ZwCB5zQUW5hlCtEMiqokhM,1071
-nui_shared_utils/__init__.py,sha256=WOpBwxBqJK_wq8l3s5TCCs4j__lMTbI7TjraTXfmH4c,11851
+nui_python_shared_utils-1.6.0.dist-info/licenses/LICENSE,sha256=vGe2mC5yLUb8toYlY3T36ZwCB5zQUW5hlCtEMiqokhM,1071
+nui_shared_utils/__init__.py,sha256=vFS5oaw7_iHgHnLRifDYYyb7au6G2nYJ6ztqMs-hrQs,13289
 nui_shared_utils/base_client.py,sha256=I1lKQGhrKyvujV2zps0TrtEdPDqMCwRQaIh6ceGONXQ,11016
 nui_shared_utils/cli.py,sha256=JJpSoQWKvAz4b8cO30yFNi5vY9jmqrCHzbFpvrVTkbU,8747
 nui_shared_utils/cloudwatch_metrics.py,sha256=p0J-s63UJDGK14TRGt0k8yoj7aWSSM55UKdgcuaT1dU,12098
@@ -37,17 +37,21 @@ nui_shared_utils/llm.py,sha256=adNUO8Qbeb3__GvpROxWSUERkpiazKiGJFE5xzU5eZ0,8351
 nui_shared_utils/log_processors.py,sha256=x5gz1LEkbmoMCZ3ZB6q-mbn26dSzwh1trz4wSkBGxqk,5538
 nui_shared_utils/powertools_helpers.py,sha256=qc-lYLfY-QX20gALfPOJcihrtJhiCU-fJW8rsPT7LJE,11037
 nui_shared_utils/secrets_helper.py,sha256=_jQkMWzjayZ8vvyHG2yuuxur-SQYUUONfcsZ0MF8qOw,7025
-nui_shared_utils/slack_client.py,sha256=_qR7Q1GU7gvYhUxiaBxEohhoEYznqr8kz9enGfeDtn8,24759
+nui_shared_utils/slack_client.py,sha256=wXE0D4T4FWt1giz9Xmp71RustpaJYeLAZo6JxulDVko,25550
 nui_shared_utils/slack_formatter.py,sha256=95g6XfAJst7RVhd0M0ahiF3gWWW5j96WYkCg5tLS6Zg,10894
 nui_shared_utils/snowflake_client.py,sha256=e3jH6_2mMqbtP1TE8-mUFFYrfGW9eqUPT5tlcgeCqx8,16181
 nui_shared_utils/timezone.py,sha256=TvtSZV7w3Vvz3NbMTcJSNbUf0ZxPwpFS-xfgye1h-4Q,3583
 nui_shared_utils/utils.py,sha256=gOpw80tZGC24QsyesR8EySRODY-TFXG8cXUvggPBsRI,8184
+nui_shared_utils/logging/__init__.py,sha256=gVC853q3ja4RhcBVbf5jKO10lFX4upfsTvkKZx4wu54,1805
+nui_shared_utils/logging/binding.py,sha256=I5VNmtf4U2yt54aS4F0Djw4u22mfjwCpWvORUnJNkPU,13577
+nui_shared_utils/logging/fields.py,sha256=7jhBdyEXAJod61aKPCk68idECTAV-_BNjiABp9IRFZo,11194
+nui_shared_utils/logging/redaction.py,sha256=t-EtNhR1nK6IOae70GnieHxn_36_jYuuta32T1dMlP8,6928
 nui_shared_utils/slack_setup/__init__.py,sha256=OElyS3xk4F_YKH5uUUTDpN0ah1dOO3e52muIPjAMPW8,320
 nui_shared_utils/slack_setup/channel_creator.py,sha256=0gyCBIS0EC96SVn1Z2dD4Fpzk0xNdHSWyiCxoUQIUe8,10667
 nui_shared_utils/slack_setup/channel_definitions.py,sha256=atfz5ZhpqefOeLh1gShbWd-TLzgjPmhv95bfuTdmZog,5676
 nui_shared_utils/slack_setup/setup_helpers.py,sha256=pzzXMs12GI9sdZttWeYzgLPgC0xqPz7ZLHM1GUPNNXc,7219
-nui_python_shared_utils-1.4.1.dist-info/METADATA,sha256=jYHBmx1R9T3PEGS2aLOY0DvcuZFm7DuebJL4VaV3z_c,22632
-nui_python_shared_utils-1.4.1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-nui_python_shared_utils-1.4.1.dist-info/entry_points.txt,sha256=TrJ3Z4kz3oXfy6InXn20jHu7rnTwo-4EA0KRFX3fkL0,66
-nui_python_shared_utils-1.4.1.dist-info/top_level.txt,sha256=sNceq5okmEB54L-gm4a0OgGhnPlU62zYmlUoXKn-Fa8,41
-nui_python_shared_utils-1.4.1.dist-info/RECORD,,
+nui_python_shared_utils-1.6.0.dist-info/METADATA,sha256=SBa2JHGtMr1R5x1PTfCp0WBkVYZB3XnLqcPIwUgYOXM,22632
+nui_python_shared_utils-1.6.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+nui_python_shared_utils-1.6.0.dist-info/entry_points.txt,sha256=TrJ3Z4kz3oXfy6InXn20jHu7rnTwo-4EA0KRFX3fkL0,66
+nui_python_shared_utils-1.6.0.dist-info/top_level.txt,sha256=sNceq5okmEB54L-gm4a0OgGhnPlU62zYmlUoXKn-Fa8,41
+nui_python_shared_utils-1.6.0.dist-info/RECORD,,

{nui_python_shared_utils-1.4.1.dist-info → nui_python_shared_utils-1.6.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (82.0.1)
+Generator: setuptools (83.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

nui_shared_utils/__init__.py

@@ -120,6 +120,25 @@ _LAZY_EXPORTS = {
     "build_anthropic_client": ("llm", "build_anthropic_client"),
     "call_tool": ("llm", "call_tool"),
     "call_text": ("llm", "call_text"),
+    # Structured logging contract (stdlib-only field registry + binders)
+    "F": ("logging", "F"),
+    "FIELDS": ("logging", "FIELDS"),
+    "Field": ("logging", "Field"),
+    "validate_fields": ("logging", "validate_fields"),
+    "validate_registry": ("logging", "validate_registry"),
+    "redaction_keys": ("logging", "redaction_keys"),
+    "alias_map": ("logging", "alias_map"),
+    "default_redactor": ("logging", "default_redactor"),
+    "powertools_binder": ("logging", "powertools_binder"),
+    "stdlib_binder": ("logging", "stdlib_binder"),
+    "PowertoolsBinder": ("logging", "PowertoolsBinder"),
+    "StdlibBinder": ("logging", "StdlibBinder"),
+    "bind_request": ("logging", "bind_request"),
+    "bind_user": ("logging", "bind_user"),
+    "bind_build_context": ("logging", "bind_build_context"),
+    "log_exception": ("logging", "log_exception"),
+    "request_body_fields": ("logging", "request_body_fields"),
+    "response_output_fields": ("logging", "response_output_fields"),
 }
 
 # Submodules that are optional integrations; ImportError during lazy load
@@ -275,6 +294,26 @@ if TYPE_CHECKING:
         validate_jwt,
     )
     from .llm import build_anthropic_client, call_text, call_tool
+    from .logging import (
+        FIELDS,
+        F,
+        Field,
+        PowertoolsBinder,
+        StdlibBinder,
+        alias_map,
+        bind_build_context,
+        bind_request,
+        bind_user,
+        default_redactor,
+        log_exception,
+        powertools_binder,
+        redaction_keys,
+        request_body_fields,
+        response_output_fields,
+        stdlib_binder,
+        validate_fields,
+        validate_registry,
+    )
     from . import slack_setup
 
 

nui_shared_utils/logging/__init__.py

@@ -0,0 +1,68 @@
+"""Structured logging contract: canonical field registry + backend-agnostic binders.
+
+The fleet's dotted, snake_case logging vocabulary and the helpers that emit it.
+Stdlib-only, so importing this package pulls in no third-party dependency (the
+Powertools binder calls ``append_keys`` / ``remove_keys`` on a logger the caller
+constructs; it never imports ``aws_lambda_powertools`` itself).
+
+Typical use::
+
+    from nui_shared_utils.logging import F, stdlib_binder, bind_request, bind_user
+
+    binder = stdlib_binder(logger)
+    with binder.request_scope():
+        bind_request(binder, method="POST", path="/orders", request_id=rid)
+        bind_user(binder, id=u.id, email=u.email, org_id=u.org_id)
+        logger.info("response", extra={F.RESPONSE_STATUS: 200})
+
+See ``docs/guides/structured-logging-contract.md`` for the full guide and
+``docs/decisions/ADR-001-structured-logging-field-vocabulary.md`` for the rationale.
+"""
+
+from .binding import (
+    PowertoolsBinder,
+    StdlibBinder,
+    bind_build_context,
+    bind_request,
+    bind_user,
+    log_exception,
+    powertools_binder,
+    request_body_fields,
+    response_output_fields,
+    stdlib_binder,
+)
+from .fields import (
+    FIELDS,
+    F,
+    Field,
+    alias_map,
+    redaction_keys,
+    validate_fields,
+    validate_registry,
+)
+from .redaction import MASK, default_redactor
+
+__all__ = [
+    # registry
+    "F",
+    "FIELDS",
+    "Field",
+    "validate_fields",
+    "validate_registry",
+    "redaction_keys",
+    "alias_map",
+    # redaction
+    "default_redactor",
+    "MASK",
+    # binders + helpers
+    "powertools_binder",
+    "stdlib_binder",
+    "PowertoolsBinder",
+    "StdlibBinder",
+    "bind_request",
+    "bind_user",
+    "bind_build_context",
+    "log_exception",
+    "request_body_fields",
+    "response_output_fields",
+]

nui_shared_utils/logging/binding.py

@@ -0,0 +1,355 @@
+"""Backend-agnostic, request-scoped binding helpers for structured logging.
+
+Two logging backends are in use across the fleet and this module drives both from
+one call site:
+
+- **Powertools** (:func:`powertools_binder`): persistent context via
+  ``Logger.append_keys`` / ``remove_keys``. Used by short-lived Lambda functions.
+- **stdlib** (:func:`stdlib_binder`): a ``contextvars``-backed ``logging.Filter``
+  that injects bound fields onto each record. Used by long-running services on the
+  ``pythonjsonlogger`` path, which has no native persistent context.
+
+Both binders are **request-scoped**: :meth:`request_scope` sets a ``contextvars``
+token on entry and unbinds on exit, so bound context cannot leak across concurrent
+requests on a long-running service or across warm Lambda invocations. ``contextvars``
+(not thread-locals) so isolation holds under ``asyncio`` too. Wire ``request_scope``
+into the request-end tween / handler ``finally``.
+
+The backend is chosen **once at construction**, not by ``hasattr`` duck-typing
+(which breaks on ``LoggerAdapter`` and other wrappers).
+
+    binder = powertools_binder(logger)   # or stdlib_binder(logger)
+    with binder.request_scope():
+        bind_request(binder, method="POST", path=p, request_id=rid)
+        bind_user(binder, id=u.id, email=u.email, org_id=u.org_id)
+        logger.info("response", extra={F.RESPONSE_STATUS: 200})   # carries bound keys
+    # on exit: bound keys removed, no cross-request / cross-invocation leakage
+
+Values are always emitted as dotted keys (``request.method``), never nested dicts.
+"""
+
+from __future__ import annotations
+
+import contextvars
+import json
+import logging
+import traceback
+from contextlib import contextmanager
+from typing import Any, Dict, Iterable, Iterator, Mapping, Optional
+
+from .fields import F
+from .redaction import default_redactor
+
+# Monotonic counter so each binder gets a distinct ContextVar name (ContextVar
+# names are for debugging only, but distinct names avoid confusion in tracebacks).
+_binder_seq = 0
+
+
+def _next_seq() -> int:
+    global _binder_seq
+    _binder_seq += 1
+    return _binder_seq
+
+
+def _drop_none(mapping: Mapping[str, Any]) -> Dict[str, Any]:
+    """Return ``mapping`` without keys whose value is ``None``."""
+    return {k: v for k, v in mapping.items() if v is not None}
+
+
+class PowertoolsBinder:
+    """Request-scoped binder over an AWS Powertools ``Logger``.
+
+    Binding calls ``logger.append_keys``; :meth:`request_scope` tracks the keys
+    bound within the scope and calls ``logger.remove_keys`` on exit so a warm
+    invocation does not inherit the previous request's context.
+    """
+
+    def __init__(self, logger: Any):
+        self.logger = logger
+        self._scope_keys: contextvars.ContextVar[Optional[set]] = contextvars.ContextVar(
+            f"nui_log_pt_scope_{_next_seq()}", default=None
+        )
+
+    def bind(self, mapping: Mapping[str, Any]) -> None:
+        """Append the given dotted keys to the logger context (``None`` dropped)."""
+        clean = _drop_none(mapping)
+        if not clean:
+            return
+        self.logger.append_keys(**clean)
+        tracked = self._scope_keys.get()
+        if tracked is not None:
+            tracked.update(clean.keys())
+
+    def unbind(self, keys: Iterable[str]) -> None:
+        """Remove the given keys from the logger context."""
+        keys = list(keys)
+        if keys:
+            self.logger.remove_keys(keys)
+
+    @contextmanager
+    def request_scope(self) -> Iterator["PowertoolsBinder"]:
+        """Scope bound keys to the block and restore prior state on exit.
+
+        Keys bound inside the block are removed on exit, but a key that already held
+        a process-level value before the scope (e.g. build context bound at startup
+        and then rebound in-request) is restored to that prior value rather than
+        removed. Restoration uses the logger's ``get_current_keys`` when available;
+        without it, scoped keys are simply removed.
+        """
+        get_keys = getattr(self.logger, "get_current_keys", None)
+        prior = dict(get_keys()) if callable(get_keys) else None
+        token = self._scope_keys.set(set())
+        try:
+            yield self
+        finally:
+            scoped = self._scope_keys.get() or set()
+            if scoped:
+                if prior is not None:
+                    added = [k for k in scoped if k not in prior]
+                    restore = {k: prior[k] for k in scoped if k in prior}
+                    if added:
+                        self.logger.remove_keys(added)
+                    if restore:
+                        self.logger.append_keys(**restore)
+                else:
+                    self.logger.remove_keys(list(scoped))
+            self._scope_keys.reset(token)
+
+
+class _ContextVarLogFilter(logging.Filter):
+    """A ``logging.Filter`` that injects the current bound fields onto each record.
+
+    Reads the binder's ``contextvars`` value at emit time, so concurrent requests
+    (each in its own task / thread context) see only their own bound fields. An
+    explicit per-call ``extra=`` value already on the record wins (bound context
+    never clobbers it).
+    """
+
+    def __init__(self, ctx: "contextvars.ContextVar[Optional[Dict[str, Any]]]"):
+        super().__init__()
+        self._ctx = ctx
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        fields = self._ctx.get()
+        if fields:
+            for key, value in fields.items():
+                if not hasattr(record, key):
+                    setattr(record, key, value)
+        return True
+
+
+class StdlibBinder:
+    """Request-scoped binder over a standard-library ``logging.Logger``.
+
+    Bound fields live in a ``contextvars.ContextVar`` and are injected onto every
+    record by a filter installed on the logger. :meth:`request_scope` snapshots the
+    context on entry and resets it on exit, so a long-running service does not bleed
+    one request's context into another (async-safe under ``contextvars``).
+    """
+
+    def __init__(self, logger: logging.Logger):
+        self.logger = logger
+        self._ctx: contextvars.ContextVar[Optional[Dict[str, Any]]] = contextvars.ContextVar(
+            f"nui_log_std_ctx_{_next_seq()}", default=None
+        )
+        self._filter = _ContextVarLogFilter(self._ctx)
+        self.install_on(logger)
+
+    def install_on(self, target: Any) -> None:
+        """Attach the context filter to a ``Logger`` or ``Handler`` (idempotent).
+
+        A logger's *filters* run only for records emitted via that exact logger, not
+        for records propagated up from child loggers (``getLogger("svc.module")``).
+        A *handler's* filters, by contrast, run for every record that reaches the
+        handler, including propagated ones. So the filter is installed on the
+        logger (direct emits) and on the effective handlers up the hierarchy (child
+        emits). The filter is a no-op outside a ``request_scope`` (empty context),
+        so attaching it to a shared / root handler is safe.
+
+        Call this explicitly for handlers configured *after* the binder is built, or
+        when your JSON handler lives on a logger other than the one passed here.
+        """
+        if isinstance(target, logging.Handler):
+            if self._filter not in target.filters:
+                target.addFilter(self._filter)
+            return
+        if self._filter not in target.filters:
+            target.addFilter(self._filter)
+        node: Optional[logging.Logger] = target
+        while node is not None:
+            for handler in node.handlers:
+                if self._filter not in handler.filters:
+                    handler.addFilter(self._filter)
+            if not node.propagate:
+                break
+            node = node.parent
+
+    def bind(self, mapping: Mapping[str, Any]) -> None:
+        """Merge the given dotted keys into the current context (``None`` dropped)."""
+        clean = _drop_none(mapping)
+        if not clean:
+            return
+        current = self._ctx.get()
+        merged = dict(current) if current else {}
+        merged.update(clean)
+        self._ctx.set(merged)
+
+    def unbind(self, keys: Iterable[str]) -> None:
+        """Remove the given keys from the current context."""
+        current = self._ctx.get()
+        if not current:
+            return
+        drop = set(keys)
+        self._ctx.set({k: v for k, v in current.items() if k not in drop})
+
+    def current_fields(self) -> Dict[str, Any]:
+        """Return a snapshot of the currently-bound fields (introspection / tests)."""
+        current = self._ctx.get()
+        return dict(current) if current else {}
+
+    @contextmanager
+    def request_scope(self) -> Iterator["StdlibBinder"]:
+        """Isolate binds made inside the block; restore prior context on exit."""
+        current = self._ctx.get()
+        token = self._ctx.set(dict(current) if current else {})
+        try:
+            yield self
+        finally:
+            self._ctx.reset(token)
+
+
+def powertools_binder(logger: Any) -> PowertoolsBinder:
+    """Build a request-scoped binder over an AWS Powertools ``Logger``."""
+    return PowertoolsBinder(logger)
+
+
+def stdlib_binder(logger: logging.Logger) -> StdlibBinder:
+    """Build a request-scoped binder over a standard-library ``logging.Logger``."""
+    return StdlibBinder(logger)
+
+
+# --- Field-specific binding helpers (callers never hand-type a dotted string) ---
+
+
+def bind_request(
+    binder: Any,
+    *,
+    method: Optional[str] = None,
+    path: Optional[str] = None,
+    ip: Optional[str] = None,
+    request_id: Optional[str] = None,
+) -> None:
+    """Bind request context (``request.method`` / ``.path`` / ``.ip`` / ``.id``)."""
+    binder.bind(
+        {
+            F.REQUEST_METHOD: method,
+            F.REQUEST_PATH: path,
+            F.REQUEST_IP: ip,
+            F.REQUEST_ID: request_id,
+        }
+    )
+
+
+def bind_user(
+    binder: Any,
+    *,
+    id: Optional[str] = None,
+    email: Optional[str] = None,
+    org_id: Optional[str] = None,
+    roles: Optional[Any] = None,
+) -> None:
+    """Bind principal identity (``user.id`` / ``.email`` / ``.org_id`` / ``.roles``)."""
+    binder.bind(
+        {
+            F.USER_ID: id,
+            F.USER_EMAIL: email,
+            F.USER_ORG_ID: org_id,
+            F.USER_ROLES: roles,
+        }
+    )
+
+
+def bind_build_context(
+    binder: Any,
+    *,
+    commit: Optional[str] = None,
+    branch: Optional[str] = None,
+    tag: Optional[str] = None,
+) -> None:
+    """Bind optional build metadata (``git.commit`` / ``.branch`` / ``.tag``).
+
+    Not bound automatically; the consumer decides whether build metadata is
+    available at runtime and calls this once at startup / per request.
+    """
+    binder.bind(
+        {
+            F.GIT_COMMIT: commit,
+            F.GIT_BRANCH: branch,
+            F.GIT_TAG: tag,
+        }
+    )
+
+
+def log_exception(binder: Any, exc: BaseException, message: str) -> None:
+    """Log ``exc`` at ERROR with explicit ``error.*`` fields.
+
+    ``error.stack`` is set explicitly from the traceback so the canonical field
+    always carries the trace. This uses ``logger.error`` (not ``logger.exception``)
+    deliberately: ``logger.exception`` sets ``exc_info=True``, which makes the
+    backend emit its *own* traceback field (Powertools' ``stack_trace``, stdlib's
+    ``exc_text``) on top of ``error.stack`` when called inside an ``except`` block,
+    i.e. the same trace twice under two names. With ``logger.error``, ``error.stack``
+    is the single source.
+    """
+    binder.logger.error(
+        message,
+        extra={
+            F.ERROR_TYPE: type(exc).__name__,
+            F.ERROR_MESSAGE: str(exc),
+            # 3-arg form for Python 3.9 compatibility (single-arg added in 3.10).
+            F.ERROR_STACK: "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)),
+        },
+    )
+
+
+def _redact_kwargs(allow_only: Optional[Iterable[str]], deny_extra: Optional[Iterable[str]]) -> Dict[str, Any]:
+    # ``deny_extra`` is only forwarded when set, so a custom ``redactor`` that does
+    # not accept it still works when the caller does not use it.
+    kwargs: Dict[str, Any] = {"allow_only": allow_only}
+    if deny_extra is not None:
+        kwargs["deny_extra"] = deny_extra
+    return kwargs
+
+
+def request_body_fields(
+    payload: Any,
+    *,
+    allow_only: Optional[Iterable[str]] = None,
+    deny_extra: Optional[Iterable[str]] = None,
+    redactor: Any = default_redactor,
+) -> Dict[str, str]:
+    """Redact and serialise a request body into a single ``request.body_json`` field.
+
+    Returns a mapping ready to splat into ``extra=`` (or pass to ``binder.bind``).
+    The body is redacted (deny-list by default, allow-list via ``allow_only``,
+    extended for this call via ``deny_extra``) then JSON-serialised with
+    ``default=str`` so non-serialisable values do not raise.
+    """
+    redacted = redactor(payload, **_redact_kwargs(allow_only, deny_extra))
+    return {F.REQUEST_BODY_JSON: json.dumps(redacted, default=str)}
+
+
+def response_output_fields(
+    payload: Any,
+    *,
+    allow_only: Optional[Iterable[str]] = None,
+    deny_extra: Optional[Iterable[str]] = None,
+    redactor: Any = default_redactor,
+) -> Dict[str, str]:
+    """Redact and serialise a response payload into a single ``response.output_json`` field.
+
+    The response twin of :func:`request_body_fields`, for services that log
+    unbounded response bodies.
+    """
+    redacted = redactor(payload, **_redact_kwargs(allow_only, deny_extra))
+    return {F.RESPONSE_OUTPUT_JSON: json.dumps(redacted, default=str)}

nui_shared_utils/logging/fields.py

@@ -0,0 +1,253 @@
+"""Canonical structured-logging field registry (the fleet's logging contract).
+
+This module is the single source of truth for the dotted, snake_case field names
+that structured logs use. It is deliberately **stdlib-only** so it stays cheap to
+import and light in a Lambda bundle: no elasticsearch, powertools, or third-party
+dependency is touched here.
+
+Two layers:
+
+- A **generic core** registry (:data:`FIELDS`) of namespaces every web / event
+  service has: ``request.*``, ``user.*``, ``route.*``, ``response.*``,
+  ``error.*``, ``trace.*``, ``git.*``, plus the ``target`` routing field and the
+  opaque ``request.body_json`` / ``response.output_json`` retrieval fields.
+- An **extension pattern**: domain fields (``order.*``, ``tender.*``, ``api.*``,
+  ``auth.*``, ...) are defined by each consumer in its own constants module and
+  drift-checked with :func:`validate_fields`, so the shared package stays generic.
+
+Each :class:`Field` carries a type, an indexed flag, a sensitivity, an owner, and
+any deprecated aliases. That one structure is the derive-once source for two lists
+that were otherwise hand-maintained:
+
+- :func:`redaction_keys` -- the default redaction deny set (from ``sensitivity``).
+- :func:`alias_map` -- the old-name -> canonical-name migration map (from ``aliases``).
+
+The contract is **store-neutral**: it governs how services *emit* fields, not how
+any log store indexes them. Its value is a consistent, typed field vocabulary
+across the fleet, which matters under any backend. Schema-enforcing stores turn
+field drift into loud errors; schema-less stores accept the drift silently and let
+queries and alerts skew. A producer-side registry prevents both.
+
+Wire shape is **dotted** (``request.method``), not nested dicts: dotted keys
+survive Powertools ``append_keys`` incremental binding where nested dicts clobber
+(a producer-side property, independent of the store). See
+``docs/decisions/ADR-001-structured-logging-field-vocabulary.md``.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field as _dc_field
+from typing import Dict, List, Tuple
+
+# Leaf datatypes a field may declare. These document producer intent (what a field
+# is meant to hold) and give a schema-enforcing store a type to map to; a schema-less
+# store coerces at query time, so the declared type is what keeps emitters honest.
+_KNOWN_TYPES = frozenset({"keyword", "text", "long", "boolean", "date", "geo_point"})
+# Sensitivity levels that drive default redaction.
+_SENSITIVITIES = frozenset({"none", "pii", "redact"})
+# A single dotted segment: lowercase snake_case, must start with a letter.
+_SEGMENT_RE = re.compile(r"^[a-z][a-z0-9_]*$")
+
+
+@dataclass(frozen=True)
+class Field:
+    """One canonical log field.
+
+    Attributes:
+        key: Dotted, snake_case field name (e.g. ``"request.method"``).
+        type: Leaf datatype the field is meant to hold (``keyword`` / ``text`` /
+            ``long`` / ``boolean`` / ``date`` / ``geo_point``). Producer intent that
+            keeps emitters consistent; not enforced by this module.
+        indexed: ``False`` marks a retrieval-only field: stored for lookup but not
+            meant to be queried or faceted (opaque, unbounded payloads like
+            ``request.body_json``).
+        sensitivity: ``none`` | ``pii`` | ``redact``. ``pii`` / ``redact`` fields
+            contribute their leaf name to the default redaction deny set.
+        owner: Namespace owner; ``core`` for shared fields.
+        aliases: Deprecated dotted names that map to this field, used to build the
+            migration / dual-read map.
+    """
+
+    key: str
+    type: str
+    indexed: bool = True
+    sensitivity: str = "none"
+    owner: str = "core"
+    aliases: Tuple[str, ...] = _dc_field(default=())
+
+
+class F:
+    """Field-name constants for ergonomic call sites (``F.REQUEST_METHOD``).
+
+    Every constant here has a matching entry in :data:`FIELDS`; the invariant is
+    asserted by :func:`validate_registry` at import time.
+    """
+
+    # request.*
+    REQUEST_ID = "request.id"
+    REQUEST_METHOD = "request.method"
+    REQUEST_PATH = "request.path"
+    REQUEST_IP = "request.ip"
+    REQUEST_BODY_JSON = "request.body_json"
+    # user.*
+    USER_ID = "user.id"
+    USER_EMAIL = "user.email"
+    USER_ORG_ID = "user.org_id"
+    USER_ROLES = "user.roles"
+    # response.*
+    RESPONSE_STATUS = "response.status"
+    RESPONSE_DURATION_MS = "response.duration_ms"
+    RESPONSE_OUTPUT_JSON = "response.output_json"
+    # route.*
+    ROUTE_NAME = "route.name"
+    ROUTE_PATH = "route.path"
+    # error.*
+    ERROR_TYPE = "error.type"
+    ERROR_MESSAGE = "error.message"
+    ERROR_STACK = "error.stack"
+    # trace.*
+    TRACE_ID = "trace.id"
+    TRACE_SPAN_ID = "trace.span_id"
+    # routing + build context
+    TARGET = "target"
+    GIT_COMMIT = "git.commit"
+    GIT_BRANCH = "git.branch"
+    GIT_TAG = "git.tag"
+
+
+# The canonical registry. Keyed by dotted name; the key must equal ``Field.key``.
+FIELDS: Dict[str, Field] = {
+    "request.id": Field("request.id", "keyword"),
+    "request.method": Field("request.method", "keyword"),
+    "request.path": Field("request.path", "keyword", aliases=("request.url",)),
+    "request.ip": Field("request.ip", "keyword"),
+    # Opaque, retrieval-only: unbounded caller-controlled body, redacted then
+    # serialised to one string so it cannot explode the index field count.
+    "request.body_json": Field("request.body_json", "keyword", indexed=False, sensitivity="redact"),
+    "user.id": Field("user.id", "keyword"),
+    "user.email": Field("user.email", "keyword", sensitivity="pii", aliases=("request.user",)),
+    # Tenant. company.* folds in here (a distinct company name, if ever needed,
+    # is a consumer domain field).
+    "user.org_id": Field("user.org_id", "keyword", aliases=("request.company", "request.company_id")),
+    "user.roles": Field("user.roles", "keyword"),  # array; ES arrays are implicit
+    "response.status": Field("response.status", "long"),
+    "response.duration_ms": Field(
+        "response.duration_ms",
+        "long",
+        aliases=("response.duration", "api.duration_ms", "duration.ms"),
+    ),
+    # The response twin of request.body_json (unbounded response payloads).
+    "response.output_json": Field("response.output_json", "keyword", indexed=False, sensitivity="redact"),
+    "route.name": Field("route.name", "keyword"),
+    "route.path": Field("route.path", "keyword"),
+    "error.type": Field("error.type", "keyword"),
+    "error.message": Field("error.message", "text"),
+    "error.stack": Field("error.stack", "text"),
+    "trace.id": Field("trace.id", "keyword", aliases=("request.trace_id",)),
+    "trace.span_id": Field("trace.span_id", "keyword"),
+    # ES index routing. Constrain to known values at the consumer (misroute risk).
+    "target": Field("target", "keyword"),
+    # Optional build context; consumer-bound via bind_build_context, not automatic.
+    "git.commit": Field("git.commit", "keyword"),
+    "git.branch": Field("git.branch", "keyword"),
+    "git.tag": Field("git.tag", "keyword"),
+}
+
+
+def _key_errors(key: object) -> List[str]:
+    """Return a list of naming problems for ``key`` (empty means valid).
+
+    A valid key is one or more ``.``-joined snake_case segments, each starting
+    with a lowercase letter: ``target``, ``request.method``, ``user.org_id``.
+    Rejects camelCase, hyphens, leading digits, and empty segments.
+    """
+    if not isinstance(key, str) or not key:
+        return [f"{key!r}: not a non-empty string"]
+    errors: List[str] = []
+    for seg in key.split("."):
+        if not _SEGMENT_RE.match(seg):
+            errors.append(f"{key!r}: segment {seg!r} is not lowercase snake_case")
+    return errors
+
+
+def validate_registry() -> List[str]:
+    """Validate the core :data:`FIELDS` registry, returning a list of errors.
+
+    Checks that every key matches its ``Field.key``, is dotted snake_case, has a
+    known type and sensitivity, and that aliases are globally unique and do not
+    collide with any canonical key. Also asserts every :class:`F` constant is a
+    registered field. An empty list means the registry is internally consistent.
+    """
+    errors: List[str] = []
+    seen_aliases: Dict[str, str] = {}
+    for key, fld in FIELDS.items():
+        if fld.key != key:
+            errors.append(f"{key!r}: Field.key {fld.key!r} does not match registry key")
+        errors.extend(_key_errors(key))
+        if fld.type not in _KNOWN_TYPES:
+            errors.append(f"{key!r}: unknown type {fld.type!r}")
+        if fld.sensitivity not in _SENSITIVITIES:
+            errors.append(f"{key!r}: unknown sensitivity {fld.sensitivity!r}")
+        for alias in fld.aliases:
+            errors.extend(_key_errors(alias))
+            if alias in FIELDS:
+                errors.append(f"{key!r}: alias {alias!r} collides with a canonical field key")
+            if alias in seen_aliases:
+                errors.append(f"alias {alias!r} used by both {seen_aliases[alias]!r} and {key!r}")
+            else:
+                seen_aliases[alias] = key
+    # Every F constant must be a real field.
+    for name in dir(F):
+        if name.startswith("_"):
+            continue
+        value = getattr(F, name)
+        if isinstance(value, str) and value not in FIELDS:
+            errors.append(f"F.{name} = {value!r} is not in FIELDS")
+    return errors
+
+
+def validate_fields(*modules: object) -> List[str]:
+    """Validate that a consumer's field constants are dotted snake_case names.
+
+    Pass one or more classes / modules whose public (non-underscore) string
+    attributes are log field names. Returns a list of naming errors (empty means
+    all valid), so a consumer's CI can assert ``validate_fields(OrderFields) == []``
+    and catch drift from the shared convention.
+    """
+    errors: List[str] = []
+    for mod in modules:
+        label = getattr(mod, "__name__", type(mod).__name__)
+        for name in dir(mod):
+            if name.startswith("_"):
+                continue
+            value = getattr(mod, name)
+            if not isinstance(value, str):
+                continue
+            errors.extend(f"{label}.{name}: {err}" for err in _key_errors(value))
+    return errors
+
+
+def redaction_keys() -> frozenset:
+    """Return the leaf names of ``pii`` / ``redact`` fields for the deny set.
+
+    These extend the redactor's base pattern list so that any payload key with a
+    sensitive name (e.g. ``email`` from ``user.email``) is masked by default.
+    """
+    return frozenset(fld.key.split(".")[-1] for fld in FIELDS.values() if fld.sensitivity in ("pii", "redact"))
+
+
+def alias_map() -> Dict[str, str]:
+    """Return the ``old_name -> canonical_key`` migration map from all aliases."""
+    mapping: Dict[str, str] = {}
+    for fld in FIELDS.values():
+        for alias in fld.aliases:
+            mapping[alias] = fld.key
+    return mapping
+
+
+# Guard the registry at import: a malformed core registry is a bug that should
+# surface the moment the logging package is first used, not silently at query time.
+_REGISTRY_ERRORS = validate_registry()
+if _REGISTRY_ERRORS:
+    raise ValueError("Invalid logging field registry:\n  " + "\n  ".join(_REGISTRY_ERRORS))

nui_shared_utils/logging/redaction.py

@@ -0,0 +1,189 @@
+"""Recursive redaction for opaque log payloads (deny-list default, allow-list opt-in).
+
+Bodies and response payloads carry unbounded, caller-controlled keys, so they are
+scrubbed before being serialised into ``request.body_json`` / ``response.output_json``.
+Two strategies, layered:
+
+- **Deny-list (default)**: a base set of sensitive key-name patterns (password,
+  token, secret, authorization, api_key, cookie, ...) plus the ``pii`` / ``redact``
+  leaf names from the field registry, matched recursively and masked. Suitable for
+  general debug payloads where most keys are legitimate and only known-sensitive
+  ones must go.
+- **Allow-list (opt-in via ``allow_only``)**: only the named keys survive, every
+  other key is masked. PII-heavy surfaces (auth, profile, anything logging headers
+  or authorization) should pass ``allow_only`` so the payload is a known safe
+  subset rather than "everything minus the deny-list".
+
+The redactor never mutates the caller's structure: it rebuilds the dict / list
+spine and shares only leaf scalars (which are treated as immutable and are
+serialised with ``json.dumps(default=str)`` downstream).
+
+Documented v1 gaps (not silently accepted, tracked for v2):
+
+- **Value-embedded secrets**: a bearer token or JWT inside a URL string, or a
+  secret in a free-text value, is not caught (only key *names* are matched).
+- **Leaky custom objects**: an object whose ``__str__`` / ``__repr__`` exposes a
+  secret is serialised as-is.
+
+Value-pattern masking (matching secret-shaped *values*, not just key names) is v2.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, Iterable, Optional
+
+from .fields import redaction_keys
+
+#: Mask substituted for redacted values.
+MASK = "***REDACTED***"
+
+#: Base deny-list of sensitive key-name patterns. Deny matching is by whole
+#: snake_case *word* (or phrase), so header variants like ``X-Api-Key`` and
+#: camelCase like ``accessToken`` match, while ``mapping`` / ``author`` /
+#: ``wildcard`` do NOT falsely match ``pin`` / ``auth`` / ``card``. Extend per
+#: call via ``deny_extra`` rather than editing this set.
+_BASE_DENY = frozenset(
+    {
+        "password",
+        "passwd",
+        "pwd",
+        "secret",
+        "token",
+        "access_token",
+        "refresh_token",
+        "id_token",
+        "authorization",
+        "auth",
+        "jwt",
+        "bearer",
+        "api_key",
+        "apikey",
+        "access_key",
+        "secret_key",
+        "private_key",
+        "session_id",
+        "cookie",
+        "set_cookie",
+        "headers",
+        "credentials",
+        "card_number",
+        "cvv",
+        "cvc",
+        "ssn",
+    }
+)
+
+#: High-signal secret tokens also matched as a plain substring of the
+#: separator-stripped key, so all-lowercase concatenations without a delimiter
+#: (``accesstoken``, ``apitoken``, ``privatekey``) are caught too. Kept to long,
+#: unambiguous words to avoid masking innocent keys (``key`` is deliberately NOT
+#: here: it would mask ``monkey`` / ``primary_key``).
+_SUBSTRING_DENY = frozenset(
+    {
+        "password",
+        "passwd",
+        "secret",
+        "token",
+        "authorization",
+        "apikey",
+        "credential",
+        "privatekey",
+        "accesskey",
+        "secretkey",
+    }
+)
+
+_NON_ALNUM = re.compile(r"[^a-z0-9]")
+_CAMEL_1 = re.compile(r"(?<=[a-z0-9])(?=[A-Z])")
+_CAMEL_2 = re.compile(r"(?<=[A-Z])(?=[A-Z][a-z])")
+_SEP = re.compile(r"[-.\s]+")
+_MULTI_US = re.compile(r"_+")
+
+
+def _to_snake(name: object) -> str:
+    """Normalise a key to lowercase snake_case (camelCase and ``-`` / ``.`` unified).
+
+    ``apiKey`` / ``X-Api-Key`` / ``api.key`` all normalise to ``api_key``, so the
+    matcher is agnostic to the caller's naming style.
+    """
+    text = str(name)
+    text = _CAMEL_1.sub("_", text)
+    text = _CAMEL_2.sub("_", text)
+    text = _SEP.sub("_", text.lower())
+    return _MULTI_US.sub("_", text).strip("_")
+
+
+def _in_deny(snake_key: str, deny: frozenset) -> bool:
+    """True if ``snake_key`` matches a deny phrase.
+
+    Two layers:
+
+    - Whole-word / phrase match on ``_``-delimited words (avoids short-token false
+      positives: ``card`` matches ``card_number`` and ``card`` but not ``wildcard``).
+    - Plain substring match for the curated high-signal tokens in
+      :data:`_SUBSTRING_DENY`, over the separator-stripped key, so all-lowercase
+      concatenations (``accesstoken``, ``apitoken``) are caught too.
+    """
+    for phrase in deny:
+        if (
+            snake_key == phrase
+            or snake_key.startswith(phrase + "_")
+            or snake_key.endswith("_" + phrase)
+            or ("_" + phrase + "_") in snake_key
+        ):
+            return True
+    collapsed = _NON_ALNUM.sub("", snake_key)
+    return any(token in collapsed for token in _SUBSTRING_DENY)
+
+
+def _build_deny(deny_extra: Optional[Iterable[str]]) -> frozenset:
+    keys = set(_BASE_DENY) | set(redaction_keys())
+    if deny_extra:
+        keys |= set(deny_extra)
+    return frozenset(_to_snake(k) for k in keys)
+
+
+def _redact(value: object, deny: frozenset, allow: Optional[frozenset]) -> object:
+    if isinstance(value, dict):
+        out: Dict[Any, Any] = {}
+        for key, val in value.items():
+            snake_key = _to_snake(key)
+            if _in_deny(snake_key, deny):
+                out[key] = MASK
+            elif allow is not None and snake_key not in allow:
+                out[key] = MASK
+            else:
+                out[key] = _redact(val, deny, allow)
+        return out
+    if isinstance(value, (list, tuple)):
+        return [_redact(item, deny, allow) for item in value]
+    return value
+
+
+def default_redactor(
+    payload: object,
+    *,
+    allow_only: Optional[Iterable[str]] = None,
+    deny_extra: Optional[Iterable[str]] = None,
+) -> object:
+    """Return a redacted copy of ``payload`` without mutating the original.
+
+    Descends dicts and lists. In the default deny-list mode, keys whose name
+    matches the base pattern set or a registry-derived sensitive leaf (by whole
+    snake_case word) are masked. Passing ``allow_only`` switches to allow-list
+    mode: only keys whose normalised name exactly equals a named key survive at any
+    depth. The deny-list still applies as a backstop, so a key that is both allowed
+    and sensitive is still masked (identity fields belong in ``bind_user``, not in a
+    redacted body). Non-container payloads (str, int, ``None``, ...) pass through
+    unchanged.
+
+    Args:
+        payload: The structure to redact (typically a request / response body).
+        allow_only: If given, keep only these key names (exact, normalised); mask
+            the rest. Use on PII-heavy routes.
+        deny_extra: Extra sensitive key names to add to the deny-list for this call.
+    """
+    deny = _build_deny(deny_extra)
+    allow = frozenset(_to_snake(k) for k in allow_only) if allow_only is not None else None
+    return _redact(payload, deny, allow)

nui_shared_utils/slack_client.py

@@ -388,7 +388,9 @@ class SlackClient(BaseClient, ServiceHealthMixin):
         blocks: Optional[List[Dict]] = None,
         attachments: Optional[List[Dict]] = None,
         include_lambda_header: bool = True,
-        event_type: Optional[str] = None
+        event_type: Optional[str] = None,
+        unfurl_links: Optional[bool] = None,
+        unfurl_media: Optional[bool] = None,
     ) -> bool:
         """
         Send message to Slack channel with standardized error handling.
@@ -400,6 +402,10 @@ class SlackClient(BaseClient, ServiceHealthMixin):
             attachments: Legacy attachment objects (supports color sidebars)
             include_lambda_header: Whether to include context header
             event_type: Optional event type label for header (e.g., "Scheduled", "API", "SQS")
+            unfurl_links: Override Slack link unfurling. None leaves Slack's default;
+                False suppresses link preview cards (links stay clickable)
+            unfurl_media: Override Slack media unfurling. None leaves Slack's default;
+                False suppresses media preview cards
 
         Returns:
             True if successful, False otherwise
@@ -419,12 +425,19 @@ class SlackClient(BaseClient, ServiceHealthMixin):
             else:
                 blocks_with_header = blocks
 
-            response = self._service_client.chat_postMessage(
+            kwargs = dict(
                 channel=channel,
                 text=text,
                 blocks=blocks_with_header,
-                attachments=attachments
+                attachments=attachments,
             )
+            # Forward unfurl controls only when explicitly set, so existing callers
+            # keep Slack's default behavior (and their call assertions stay unchanged).
+            if unfurl_links is not None:
+                kwargs["unfurl_links"] = unfurl_links
+            if unfurl_media is not None:
+                kwargs["unfurl_media"] = unfurl_media
+            response = self._service_client.chat_postMessage(**kwargs)
 
             if response["ok"]:
                 log.info(