alpha-engine-lib 0.32.0__py3-none-any.whl

alpha_engine_lib/alerts.py

@@ -0,0 +1,576 @@
+"""
+Unified failure-surveillance fan-out for Alpha Engine modules.
+
+Consolidation substrate for the **"fire an operator alert from a failure
+site"** pattern that has appeared inline across the fleet:
+
+* :file:`alpha-engine/infrastructure/health_checker.sh` — raw ``curl`` to
+  Telegram bot API
+* :file:`alpha-engine-data/infrastructure/lambdas/changelog-incident-mirror/deploy.sh`
+  — raw ``aws sns publish`` to ``alpha-engine-alerts``
+* ROADMAP L116/L117 — names 5 more Lambda-deploying repos that need the
+  same canary-rollback alert primitive ("Mirror in all 5 Lambda-deploying
+  repos … same recurrence class as ``feedback_env_regression_recurs_per_repo_spot_script``
+  — fix forward across all repos in one pass, not per-repo at incident time")
+
+Per the ``~/Development/CLAUDE.md`` SOTA / institutional-approach rule
+(sub-sub-rule: lift to lib when ≥2 consumers exist), this module is the
+canonical Python primitive backing all consumers. Bash callers reach it
+via the CLI entry (``python -m alpha_engine_lib.alerts publish ...``) —
+mirrors the :mod:`alpha_engine_lib.transparency` ``--cadence daily/weekly``
+CLI convention.
+
+**Public API:**
+
+- :func:`publish` — fan-out to both SNS (``alpha-engine-alerts`` topic →
+  email) and Telegram (``@nous_ergon_alerts_bot`` channel) by default.
+  Each channel is independently best-effort — failure in one does not
+  block the other. Returns a :class:`PublishResult` dataclass with the
+  per-channel outcome for caller observability.
+- CLI: ``python -m alpha_engine_lib.alerts publish --message "..."
+  --severity error --source "..."``. Designed for Bash failure-trap
+  callers (``cleanup()`` in spot dispatchers, ``deploy.sh`` rollback
+  branches). Exit code is ``0`` if *either* channel succeeded, ``1`` if
+  *both* failed.
+
+**Severity tiering.** ``severity`` is a free-form string that is
+prepended to the message (``[ERROR] ...`` / ``[WARNING] ...``) for both
+channels. Telegram pushes (``disable_notification=False``) for
+``error``/``critical``; in-channel silent for ``info``/``warning``. SNS
+delivery is identical regardless of severity — downstream subscribers
+choose how to fan out.
+
+**SNS topic resolution.** Defaults to
+``arn:aws:sns:{region}:{account_id}:alpha-engine-alerts``, with
+``region`` from ``AWS_REGION``/``AWS_DEFAULT_REGION`` (fallback
+``us-east-1``) and ``account_id`` resolved via ``sts:GetCallerIdentity``.
+Override with the ``--sns-topic-arn`` CLI flag or ``sns_topic_arn``
+kwarg.
+
+**Failure behavior.** Never raises. SNS errors (boto3 ``ClientError``,
+network) and Telegram errors both log at WARNING and return a
+:class:`PublishResult` with the failed channel marked ``ok=False``. This
+is by design — the caller is already in a failure path; secondary
+surveillance failure must not mask the primary error.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+import sys
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
+from typing import Final
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_SNS_TOPIC_NAME: Final[str] = "alpha-engine-alerts"
+DEFAULT_REGION: Final[str] = "us-east-1"
+SEVERITY_PUSH: Final[frozenset[str]] = frozenset({"error", "critical"})
+
+# ── Dedup (v0.24.0) ──────────────────────────────────────────────────────────
+# When the caller passes a ``dedup_key``, ``publish`` writes a marker at
+# ``s3://{dedup_bucket}/{DEDUP_MARKER_PREFIX}/{sha1(dedup_key)[:16]}.json``
+# after the first successful publish. Subsequent calls with the same
+# ``dedup_key`` within ``dedup_window_min`` minutes find the marker and
+# skip the publish. See the :func:`publish` docstring.
+DEFAULT_DEDUP_BUCKET: Final[str] = "alpha-engine-research"
+DEDUP_MARKER_PREFIX: Final[str] = "_alerts/_dedup"
+DEFAULT_DEDUP_WINDOW_MIN: Final[int] = 60
+
+
+@dataclass
+class ChannelResult:
+    """Per-channel outcome from a :func:`publish` call."""
+
+    ok: bool
+    detail: str = ""
+
+
+@dataclass
+class PublishResult:
+    """Aggregated outcome from a :func:`publish` call.
+
+    ``sns`` and ``telegram`` are independent — a publish may succeed in
+    one channel and fail in the other. :attr:`any_ok` is the typical
+    caller gate (success = at least one channel delivered the alert);
+    :attr:`all_ok` is the strict variant for callers that want both.
+
+    When the caller passes ``dedup_key`` and an earlier publish for the
+    same key is still within window, :attr:`dedup_skipped` is True and
+    neither channel is attempted; :attr:`any_ok` still reports True
+    (the alert is logically in the operator's hands by virtue of the
+    earlier successful publish).
+    """
+
+    sns: ChannelResult = field(default_factory=lambda: ChannelResult(ok=False, detail="not attempted"))
+    telegram: ChannelResult = field(default_factory=lambda: ChannelResult(ok=False, detail="not attempted"))
+    dedup_skipped: bool = False
+    dedup_reason: str = ""
+
+    @property
+    def any_ok(self) -> bool:
+        if self.dedup_skipped:
+            return True
+        return self.sns.ok or self.telegram.ok
+
+    @property
+    def all_ok(self) -> bool:
+        if self.dedup_skipped:
+            return True
+        return self.sns.ok and self.telegram.ok
+
+
+def _resolve_sns_topic_arn(explicit: str | None) -> str | None:
+    """Return the SNS topic ARN, resolving from env + STS if not explicit."""
+    if explicit:
+        return explicit
+    region = (
+        os.environ.get("AWS_REGION")
+        or os.environ.get("AWS_DEFAULT_REGION")
+        or DEFAULT_REGION
+    )
+    try:
+        import boto3
+
+        account_id = boto3.client("sts", region_name=region).get_caller_identity()["Account"]
+    except Exception as exc:  # boto3 missing, STS unreachable, creds bad
+        logger.warning("alerts.publish: SNS topic ARN resolution failed: %s", exc)
+        return None
+    return f"arn:aws:sns:{region}:{account_id}:{DEFAULT_SNS_TOPIC_NAME}"
+
+
+def _format_message(message: str, severity: str, source: str | None) -> str:
+    """Prepend severity tag + source prefix to the message body."""
+    tag = f"[{severity.upper()}]"
+    if source:
+        return f"{tag} {source}: {message}"
+    return f"{tag} {message}"
+
+
+def _publish_sns(arn: str, message: str, subject: str | None = None) -> ChannelResult:
+    try:
+        import boto3
+
+        region = arn.split(":")[3] if ":" in arn else DEFAULT_REGION
+        client = boto3.client("sns", region_name=region)
+        kwargs: dict = {"TopicArn": arn, "Message": message}
+        if subject:
+            # SNS subject is limited to 100 chars + ASCII + no newlines.
+            cleaned = subject.replace("\n", " ").replace("\r", " ")[:100]
+            kwargs["Subject"] = cleaned
+        resp = client.publish(**kwargs)
+        return ChannelResult(ok=True, detail=resp.get("MessageId", "<no id>"))
+    except Exception as exc:
+        logger.warning("alerts.publish: SNS publish failed: %s", exc)
+        return ChannelResult(ok=False, detail=f"sns error: {exc!r}")
+
+
+def _publish_telegram(message: str, severity: str) -> ChannelResult:
+    try:
+        from alpha_engine_lib.telegram import send_message
+
+        # Push for error/critical, silent in-channel for info/warning.
+        silent = severity.lower() not in SEVERITY_PUSH
+        ok = send_message(message, disable_notification=silent)
+        return ChannelResult(ok=bool(ok), detail="sent" if ok else "send_message returned False")
+    except Exception as exc:  # send_message itself never raises, but defensive
+        logger.warning("alerts.publish: Telegram fan-out failed: %s", exc)
+        return ChannelResult(ok=False, detail=f"telegram error: {exc!r}")
+
+
+def _dedup_marker_key(dedup_key: str) -> str:
+    """Stable S3 key for a dedup_key marker.
+
+    Hashes the dedup_key so the on-disk path is opaque + bounded length
+    (S3 keys can be arbitrarily long but operator-facing S3 listings
+    are easier to read with fixed-width entries). The original
+    dedup_key is preserved inside the marker JSON body for debugging.
+    """
+    digest = hashlib.sha1(dedup_key.encode("utf-8")).hexdigest()[:16]
+    return f"{DEDUP_MARKER_PREFIX}/{digest}.json"
+
+
+def _check_dedup_marker(
+    bucket: str,
+    marker_key: str,
+    *,
+    dedup_window_min: int | None,
+) -> tuple[bool, str]:
+    """Check whether a recent publish for this dedup_key is still in window.
+
+    Returns ``(within_window, reason)``. ``within_window=True`` means
+    the caller should skip publish; ``False`` means proceed.
+
+    Fail-safe: any S3 error other than NoSuchKey returns
+    ``(False, "<error description>")`` so the caller proceeds to
+    publish. An extra alert is preferable to silently dropping a real
+    failure-surveillance event because the marker bucket was
+    unreachable.
+
+    ``dedup_window_min=None`` means "forever" — any existing marker
+    suppresses subsequent publishes indefinitely.
+    """
+    try:
+        import boto3
+        from botocore.exceptions import ClientError
+    except ImportError as exc:
+        return False, f"boto3 unavailable: {exc!r}"
+    client = boto3.client("s3")
+    try:
+        resp = client.get_object(Bucket=bucket, Key=marker_key)
+        payload = json.loads(resp["Body"].read())
+    except ClientError as exc:
+        code = exc.response.get("Error", {}).get("Code", "")
+        if code == "NoSuchKey":
+            return False, "no marker"
+        logger.warning(
+            "alerts.publish: dedup marker check errored (fail-safe to publish): %s",
+            exc,
+        )
+        return False, f"marker check error: {exc!r}"
+    except Exception as exc:  # boto3 missing, network, JSON parse
+        logger.warning(
+            "alerts.publish: dedup marker parse failed (fail-safe to publish): %s",
+            exc,
+        )
+        return False, f"marker parse error: {exc!r}"
+
+    if dedup_window_min is None:
+        return True, f"marker exists; dedup_window_min=None (forever)"
+
+    last_at_str = payload.get("last_published_at") or payload.get("first_published_at")
+    if not last_at_str:
+        return False, "marker missing timestamp"
+    try:
+        last_at = datetime.fromisoformat(last_at_str.replace("Z", "+00:00"))
+    except ValueError:
+        return False, f"marker timestamp unparseable: {last_at_str!r}"
+
+    now = datetime.now(timezone.utc)
+    elapsed = now - last_at
+    window = timedelta(minutes=dedup_window_min)
+    if elapsed < window:
+        remaining = window - elapsed
+        return True, (
+            f"within {dedup_window_min}min window "
+            f"(last published {int(elapsed.total_seconds())}s ago; "
+            f"{int(remaining.total_seconds())}s remaining)"
+        )
+    return False, f"marker expired ({int(elapsed.total_seconds())}s ago > {dedup_window_min}min)"
+
+
+def _write_dedup_marker(
+    bucket: str,
+    marker_key: str,
+    *,
+    dedup_key: str,
+    formatted_message: str,
+) -> None:
+    """Persist (or refresh) the dedup marker after a successful publish.
+
+    Read-modify-write: increments ``publish_count`` if the marker
+    already exists, otherwise starts a fresh marker. Best-effort — any
+    failure is logged at WARNING and swallowed (worst case: one
+    duplicate alert next time within the window).
+    """
+    try:
+        import boto3
+        from botocore.exceptions import ClientError
+    except ImportError as exc:
+        logger.warning(
+            "alerts.publish: dedup marker write skipped — boto3 unavailable: %s",
+            exc,
+        )
+        return
+    client = boto3.client("s3")
+    now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    # Read-modify-write so first_published_at is preserved across window
+    # refreshes; publish_count grows monotonically.
+    first_published_at = now_iso
+    publish_count = 1
+    try:
+        resp = client.get_object(Bucket=bucket, Key=marker_key)
+        prior = json.loads(resp["Body"].read())
+        first_published_at = prior.get("first_published_at", now_iso)
+        publish_count = int(prior.get("publish_count", 0)) + 1
+    except ClientError as exc:
+        code = exc.response.get("Error", {}).get("Code", "")
+        if code != "NoSuchKey":
+            # Non-fatal — fall through to write a fresh marker.
+            logger.warning(
+                "alerts.publish: dedup marker RMW read failed (writing fresh): %s",
+                exc,
+            )
+    except Exception:  # JSON parse / corrupt marker — overwrite
+        pass
+
+    payload = {
+        "dedup_key": dedup_key,
+        "first_published_at": first_published_at,
+        "last_published_at": now_iso,
+        "publish_count": publish_count,
+        "message_preview": formatted_message[:200],
+    }
+    try:
+        client.put_object(
+            Bucket=bucket,
+            Key=marker_key,
+            Body=json.dumps(payload).encode("utf-8"),
+            ContentType="application/json",
+        )
+    except Exception as exc:  # noqa: BLE001
+        logger.warning(
+            "alerts.publish: dedup marker write failed "
+            "(best-effort, swallowed; next call within window may re-publish): %s",
+            exc,
+        )
+
+
+def publish(
+    message: str,
+    *,
+    severity: str = "error",
+    source: str | None = None,
+    sns: bool = True,
+    telegram: bool = True,
+    sns_topic_arn: str | None = None,
+    dedup_key: str | None = None,
+    dedup_window_min: int | None = DEFAULT_DEDUP_WINDOW_MIN,
+    dedup_bucket: str | None = None,
+) -> PublishResult:
+    """Fan out a failure alert to the operator-surveillance channels.
+
+    Default: publish to both ``alpha-engine-alerts`` SNS (→ email) AND
+    Telegram (``@nous_ergon_alerts_bot``). Pass ``sns=False`` /
+    ``telegram=False`` to suppress individual channels (useful for
+    tests, or for callers that have a narrower target).
+
+    **Dedup** (v0.24.0). When ``dedup_key`` is provided, the call
+    checks an S3 marker at
+    ``s3://{dedup_bucket}/_alerts/_dedup/{sha1(dedup_key)[:16]}.json``.
+    If the marker exists and the last publish for that key is within
+    ``dedup_window_min`` minutes (default ``60``; ``None`` = forever),
+    the publish is suppressed and :attr:`PublishResult.dedup_skipped`
+    is True. After a successful fresh publish, the marker is written
+    (or refreshed) with an incremented ``publish_count``. Use cases:
+
+    - **One email per cost anomaly** even when ``evaluate.py`` runs
+      multiple times for the same date — pass a deterministic
+      ``dedup_key`` derived from the anomaly inputs.
+    - **One alert per Lambda canary rollback episode** even when 8
+      Lambda repos cascade-fail from one shared lib regression — pass
+      ``dedup_key=f"canary-rollback-{lib_pin_sha}"`` so the cascading
+      deploys all collapse to one operator email.
+    - **Once-per-hour throttling** on noisy WARN paths — pass any
+      stable key + leave the default 60min window.
+
+    Dedup is best-effort: any S3 error during the check falls through
+    to publish (better an extra alert than a silent drop). Marker
+    write failure after a successful publish is logged but does NOT
+    propagate (worst case is one duplicate next call within window).
+
+    :param message: The alert body. Severity tag + source prefix are
+        prepended automatically (e.g. ``"[ERROR] spot_backtest.sh: <body>"``).
+    :param severity: Free-form severity string (``error`` / ``critical``
+        push on Telegram; everything else is silent in-channel). The tag
+        is uppercased in the rendered message.
+    :param source: Optional source identifier (script path, repo, Lambda
+        name) inserted between the tag and the message body. Helps the
+        operator triage at a glance.
+    :param sns: When ``False``, skip the SNS publish entirely.
+    :param telegram: When ``False``, skip the Telegram fan-out entirely.
+    :param sns_topic_arn: Explicit topic ARN. Defaults to
+        ``arn:aws:sns:{region}:{account_id}:alpha-engine-alerts`` resolved
+        from env + STS.
+    :param dedup_key: Opaque caller-chosen string. Same key + same
+        window ⇒ at most one publish per window. ``None`` (default)
+        disables dedup entirely; legacy callers behave unchanged.
+    :param dedup_window_min: Window in minutes after which a fresh
+        publish is allowed for the same ``dedup_key``. Default
+        ``60``. Pass ``None`` for "forever" (publish once per
+        ``dedup_key`` for the lifetime of the marker bucket).
+    :param dedup_bucket: S3 bucket holding the markers. Defaults to
+        ``alpha-engine-research`` (the shared corpus bucket).
+    :returns: :class:`PublishResult` — caller can inspect per-channel
+        outcomes. :attr:`PublishResult.any_ok` is the typical success
+        gate; :attr:`PublishResult.all_ok` is the strict variant.
+        On dedup-skip, :attr:`PublishResult.dedup_skipped` is True and
+        :attr:`PublishResult.dedup_reason` explains why.
+    """
+    result = PublishResult()
+    formatted = _format_message(message, severity, source)
+
+    # ── Dedup check (pre-publish) ────────────────────────────────────────
+    marker_key: str | None = None
+    bucket = dedup_bucket or DEFAULT_DEDUP_BUCKET
+    if dedup_key:
+        marker_key = _dedup_marker_key(dedup_key)
+        within_window, reason = _check_dedup_marker(
+            bucket, marker_key, dedup_window_min=dedup_window_min,
+        )
+        if within_window:
+            result.dedup_skipped = True
+            result.dedup_reason = reason
+            result.sns = ChannelResult(ok=False, detail="suppressed by dedup")
+            result.telegram = ChannelResult(ok=False, detail="suppressed by dedup")
+            logger.info(
+                "alerts.publish: skipped publish for dedup_key=%r (%s)",
+                dedup_key, reason,
+            )
+            return result
+
+    # ── Publish ──────────────────────────────────────────────────────────
+    if sns:
+        arn = _resolve_sns_topic_arn(sns_topic_arn)
+        if arn is None:
+            result.sns = ChannelResult(ok=False, detail="topic ARN resolution failed")
+        else:
+            # SNS subject — concise header, falls back to severity tag.
+            subject = f"Alpha Engine alert [{severity.upper()}]"
+            if source:
+                subject += f" — {source}"
+            result.sns = _publish_sns(arn, formatted, subject=subject)
+
+    if telegram:
+        result.telegram = _publish_telegram(formatted, severity=severity)
+
+    # ── Dedup marker write (post-publish, only if any channel succeeded) ─
+    if marker_key and (result.sns.ok or result.telegram.ok):
+        _write_dedup_marker(
+            bucket, marker_key,
+            dedup_key=dedup_key, formatted_message=formatted,
+        )
+
+    return result
+
+
+# ─── CLI entry ──────────────────────────────────────────────────────────────
+# Designed for Bash callers that need failure surveillance from a script
+# (spot dispatcher `cleanup` traps, deploy.sh rollback branches, etc.).
+# Mirrors the :mod:`alpha_engine_lib.transparency` ``python -m`` pattern so
+# Bash callers reach this primitive without bootstrapping a full Python
+# project. Exit code is 0 if *any* channel succeeded, 1 if both failed.
+
+
+def main(argv: list[str] | None = None) -> int:
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        prog="python -m alpha_engine_lib.alerts",
+        description=(
+            "Publish a failure alert to alpha-engine's operator-surveillance "
+            "channels (SNS topic alpha-engine-alerts + Telegram). Designed "
+            "for Bash callers — exit code 0 if any channel succeeded, 1 if "
+            "both failed. Never raises."
+        ),
+    )
+    subparsers = parser.add_subparsers(dest="cmd", required=True)
+
+    pub = subparsers.add_parser("publish", help="Publish an alert message.")
+    pub.add_argument("--message", required=True, help="Alert body text.")
+    pub.add_argument(
+        "--severity",
+        default="error",
+        help=(
+            "Severity tag (default: error). 'error' and 'critical' push on "
+            "Telegram; all others are silent in-channel."
+        ),
+    )
+    pub.add_argument(
+        "--source",
+        default=None,
+        help=(
+            "Optional source identifier (script path, repo, Lambda name) "
+            "rendered between the severity tag and the message body."
+        ),
+    )
+    pub.add_argument("--no-sns", action="store_true", help="Skip SNS publish.")
+    pub.add_argument("--no-telegram", action="store_true", help="Skip Telegram fan-out.")
+    pub.add_argument(
+        "--sns-topic-arn",
+        default=None,
+        help=(
+            "Override the SNS topic ARN. Defaults to "
+            "arn:aws:sns:{region}:{account_id}:alpha-engine-alerts."
+        ),
+    )
+    pub.add_argument(
+        "--dedup-key",
+        default=None,
+        help=(
+            "Optional opaque dedup key. When set, ``publish`` checks an "
+            "S3 marker first and suppresses the alert if an earlier "
+            "publish for the same key is within --dedup-window-min. "
+            "Use for cost anomalies / canary rollback episodes / any "
+            "noisy WARN path that benefits from rate-limiting. Bash "
+            "callers typically pass a bucketed timestamp, e.g. "
+            "--dedup-key \"canary-rollback-$(date -u +%Y%m%d%H)\"."
+        ),
+    )
+    pub.add_argument(
+        "--dedup-window-min",
+        type=int,
+        default=DEFAULT_DEDUP_WINDOW_MIN,
+        help=(
+            f"Window in minutes after which a fresh publish is allowed for "
+            f"the same --dedup-key (default: {DEFAULT_DEDUP_WINDOW_MIN}). "
+            "Pass 0 for 'forever' (publish once per --dedup-key for the "
+            "lifetime of the marker bucket)."
+        ),
+    )
+    pub.add_argument(
+        "--dedup-bucket",
+        default=None,
+        help=(
+            f"S3 bucket holding the dedup markers. Defaults to "
+            f"{DEFAULT_DEDUP_BUCKET!r}."
+        ),
+    )
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(level=logging.WARNING)
+
+    # CLI convention: --dedup-window-min 0 = forever; map to None for the
+    # Python API (whose default is 60 + None=forever).
+    window_min: int | None
+    if args.dedup_window_min == 0:
+        window_min = None
+    else:
+        window_min = args.dedup_window_min
+
+    result = publish(
+        args.message,
+        severity=args.severity,
+        source=args.source,
+        sns=not args.no_sns,
+        telegram=not args.no_telegram,
+        sns_topic_arn=args.sns_topic_arn,
+        dedup_key=args.dedup_key,
+        dedup_window_min=window_min,
+        dedup_bucket=args.dedup_bucket,
+    )
+
+    # One-line status to stderr (stdout reserved for structured output if
+    # any caller starts parsing it). Bash callers can ignore.
+    if result.dedup_skipped:
+        print(
+            f"alerts.publish: dedup_skipped=True ({result.dedup_reason})",
+            file=sys.stderr,
+        )
+    else:
+        print(
+            f"alerts.publish: sns.ok={result.sns.ok} ({result.sns.detail}); "
+            f"telegram.ok={result.telegram.ok} ({result.telegram.detail})",
+            file=sys.stderr,
+        )
+
+    return 0 if result.any_ok else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())