loghunter-cli 0.1.0.dev0__py3-none-any.whl

loghunter/digest/dns.py

@@ -0,0 +1,364 @@
+"""dns summariser — orient-before-the-hunt for DNS data.
+
+The first fidelity-aware digest card: a slot set that depends on which DNS
+feed was loaded. Four slots are shared (cliff/cliff/tail/dist over columns
+present on both feeds); two are feed-specific:
+
+  - nxdomain-rate (rcode-based) — Zeek only; non-speaking on Pi-hole
+  - block-rate (event_type-based) — Pi-hole only; non-speaking on Zeek
+
+A feed-uncomputable slot returns a non-speaking ``DigestSlot`` (cells=None);
+``select_insights_and_fields`` filters it out of ``fields`` and the slot
+simply vanishes from the rendered card. No ABSENT marker, no footer text.
+
+Cliff machinery imported from conn so the two cards cannot drift on gate /
+floor / display-cap behaviour. The rate statistic — and its RATE_FLOOR
+constant — live in ``loghunter.digest._stats`` (factored once three cards
+needed an identical copy: this one, syslog, and cloudtrail). Two more
+statistics computed locally:
+
+  - tail: max/median ratio over a distribution, with an owner attribution
+  - dist: top-3 share-of-mix; orientation only, never produces an insight
+
+A row is "blocked" on Pi-hole iff event_type ∈ {gravity_blocked,
+regex_blocked} — digest computes this locally; the detector is not imported.
+"""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from loghunter.common.finding import DigestSlot
+from loghunter.digest._stats import RATE_FLOOR, _rate
+from loghunter.digest.conn import (
+    CLIFF_DISPLAY_CAP,  # noqa: F401 — re-exported for downstream symmetry
+    CLIFF_GATE,         # noqa: F401 — re-exported for downstream symmetry
+    POPULATION_FLOOR,
+    _cliff,
+    _format_ratio_cell,
+    _format_ratio_lede,
+)
+
+
+# ── Calibration constants — provisional, tunable in one place ───────────────
+
+TAIL_GATE = 3.0       # max/median ratio below this → query-length is non-speaking
+
+# Zeek emits qtype as a numeric type code; map the common ones to mnemonics
+# for display. Unmapped codes render as "TYPE<n>" so an analyst still has a
+# breadcrumb to look up.
+_ZEEK_QTYPE_MNEMONICS = {
+    1: "A", 2: "NS", 5: "CNAME", 6: "SOA", 12: "PTR",
+    15: "MX", 16: "TXT", 28: "AAAA", 33: "SRV", 65: "HTTPS", 257: "CAA",
+}
+
+_BLOCK_EVENT_TYPES = frozenset({"gravity_blocked", "regex_blocked"})
+
+
+# ── tail statistic ──────────────────────────────────────────────────────────
+
+def _tail(values: pd.Series, owner_series: pd.Series) -> tuple | None:
+    """Tail statistic: is the extreme far from the body of the distribution?
+
+    Returns ``(max_val, ratio, owner)`` when speaking, None when dashed.
+    Dashes when population < POPULATION_FLOOR, median is 0/NaN, or
+    max/median is below TAIL_GATE.
+
+    ``values`` and ``owner_series`` must share an index — the owner of the
+    max is looked up by that index.
+    """
+    cleaned = values.dropna()
+    if len(cleaned) < POPULATION_FLOOR:
+        return None
+    median = cleaned.median()
+    if pd.isna(median) or median == 0:
+        return None
+    max_val = cleaned.max()
+    if pd.isna(max_val) or max_val == 0:
+        return None
+    ratio = float(max_val) / float(median)
+    if ratio < TAIL_GATE:
+        return None
+    max_idx = cleaned.idxmax()
+    try:
+        owner = owner_series.loc[max_idx]
+    except (KeyError, ValueError):
+        return None
+    if pd.isna(owner):
+        return None
+    return int(max_val), ratio, str(owner)
+
+
+# ── dist statistic — qtype-mix, always shows ────────────────────────────────
+
+def _qtype_label(value: object, feed: str) -> str | None:
+    """Map a single qtype value to a display string.
+
+    Zeek: numeric code → mnemonic from _ZEEK_QTYPE_MNEMONICS; unmapped
+    integers → ``"TYPE<n>"``. Pi-hole: already a string mnemonic; used
+    as-is. NaN / unparseable → None (caller filters).
+    """
+    if pd.isna(value):
+        return None
+    if feed == "pihole":
+        s = str(value).strip()
+        return s if s else None
+    try:
+        code = int(value)
+    except (TypeError, ValueError):
+        s = str(value).strip()
+        return s if s else None
+    return _ZEEK_QTYPE_MNEMONICS.get(code, f"TYPE{code}")
+
+
+def _qtype_dist(qtypes: pd.Series | None, feed: str) -> str:
+    """Render top-3 qtype share string for the qtype-mix dist slot.
+
+    Two distinct fallbacks (consistency pinned by review):
+      - Missing column (qtypes is None) → "(no qtype)" (schema-presence fact)
+      - Empty / all-NaN series → "(no queries)" (data-shape fact)
+    Single-type pile → "A 100%". Mix → "A 82% · AAAA 11% · HTTPS 4%".
+    """
+    if qtypes is None:
+        return "(no qtype)"
+    labels = qtypes.map(lambda v: _qtype_label(v, feed)).dropna()
+    if labels.empty:
+        return "(no queries)"
+    counts = labels.value_counts()
+    total = int(counts.sum())
+    top_three = counts.head(3)
+    parts = [
+        f"{label} {int(round(count / total * 100))}%"
+        for label, count in top_three.items()
+    ]
+    return " · ".join(parts)
+
+
+# ── Slot computers ──────────────────────────────────────────────────────────
+
+def _slot_client_volume(frame: pd.DataFrame) -> DigestSlot:
+    """client-volume — cliff over per-src query counts."""
+    label = "client-volume"
+    if frame.empty or "src" not in frame.columns:
+        return DigestSlot(label=label, statistic="cliff")
+    counts = frame["src"].value_counts(dropna=True).sort_values(ascending=False)
+    result = _cliff(counts)
+    if result is None:
+        return DigestSlot(label=label, statistic="cliff")
+    entity, magnitude, ratio = result
+    total = len(frame)
+    share_pct = (magnitude / total * 100.0) if total > 0 else 0.0
+    entity_str = str(entity)
+    return DigestSlot(
+        label=label,
+        statistic="cliff",
+        cells=[entity_str, f"{share_pct:.0f}%", _format_ratio_cell(ratio)],
+        entity=entity_str,
+        magnitude=share_pct,
+        ratio=ratio,
+    )
+
+
+def _slot_domain_volume(frame: pd.DataFrame) -> DigestSlot:
+    """domain-volume — cliff over per-query counts."""
+    label = "domain-volume"
+    if frame.empty or "query" not in frame.columns:
+        return DigestSlot(label=label, statistic="cliff")
+    counts = frame["query"].value_counts(dropna=True).sort_values(ascending=False)
+    result = _cliff(counts)
+    if result is None:
+        return DigestSlot(label=label, statistic="cliff")
+    entity, magnitude, ratio = result
+    entity_str = str(entity)
+    return DigestSlot(
+        label=label,
+        statistic="cliff",
+        cells=[entity_str, f"{int(magnitude)}", _format_ratio_cell(ratio)],
+        entity=entity_str,
+        magnitude=magnitude,
+        ratio=ratio,
+    )
+
+
+def _slot_query_length(frame: pd.DataFrame) -> DigestSlot:
+    """query-length — tail over query character lengths; names the owner.
+
+    Cell order per brief: ``[maxlen, ratio, owner]``. The lede leads with
+    the owner, but the table row leads with the magnitude (length of
+    longest query) first.
+    """
+    label = "query-length"
+    if frame.empty or "query" not in frame.columns or "src" not in frame.columns:
+        return DigestSlot(label=label, statistic="tail")
+    queries = frame["query"].dropna().astype(str)
+    if queries.empty:
+        return DigestSlot(label=label, statistic="tail")
+    lengths = queries.str.len()
+    src_aligned = frame.loc[queries.index, "src"]
+    result = _tail(lengths, src_aligned)
+    if result is None:
+        return DigestSlot(label=label, statistic="tail")
+    max_val, ratio, owner = result
+    return DigestSlot(
+        label=label,
+        statistic="tail",
+        cells=[f"{max_val} chars", _format_ratio_cell(ratio), owner],
+        entity=owner,
+        magnitude=float(max_val),
+        ratio=ratio,
+    )
+
+
+def _slot_qtype_mix(frame: pd.DataFrame, feed: str) -> DigestSlot:
+    """qtype-mix — dist over query types; always shows."""
+    label = "qtype-mix"
+    qtypes = frame["qtype"] if "qtype" in frame.columns else None
+    rendered = _qtype_dist(qtypes, feed)
+    return DigestSlot(label=label, statistic="dist", cells=[rendered])
+
+
+def _slot_nxdomain_rate(frame: pd.DataFrame, feed: str) -> DigestSlot:
+    """nxdomain-rate — rate of NXDOMAIN (rcode == 3). Zeek only.
+
+    Non-Zeek feeds return a non-speaking slot — the summariser filters those
+    out, so the slot vanishes from the card entirely on Pi-hole.
+    """
+    label = "nxdomain-rate"
+    if feed != "zeek":
+        return DigestSlot(label=label, statistic="rate")
+    if frame.empty or "rcode" not in frame.columns or "src" not in frame.columns:
+        return DigestSlot(label=label, statistic="rate")
+    kind_mask = (frame["rcode"] == 3).fillna(False).astype(bool)
+    result = _rate(kind_mask, frame["src"])
+    if result is None:
+        return DigestSlot(label=label, statistic="rate")
+    fraction, top = result
+    pct = fraction * 100.0
+    return DigestSlot(
+        label=label,
+        statistic="rate",
+        cells=[f"{pct:.0f}% failed", top],
+        entity=top,
+        magnitude=pct,
+    )
+
+
+def _slot_block_rate(frame: pd.DataFrame, feed: str) -> DigestSlot:
+    """block-rate — rate of blocked queries (gravity_blocked / regex_blocked).
+    Pi-hole only. Block-status derivation is local; the detector is not
+    imported.
+
+    Non-Pi-hole feeds return a non-speaking slot — the summariser filters
+    those out, so the slot vanishes from the card entirely on Zeek.
+    """
+    label = "block-rate"
+    if feed != "pihole":
+        return DigestSlot(label=label, statistic="rate")
+    if frame.empty or "event_type" not in frame.columns or "query" not in frame.columns:
+        return DigestSlot(label=label, statistic="rate")
+    kind_mask = frame["event_type"].isin(_BLOCK_EVENT_TYPES).fillna(False).astype(bool)
+    result = _rate(kind_mask, frame["query"])
+    if result is None:
+        return DigestSlot(label=label, statistic="rate")
+    fraction, top = result
+    pct = fraction * 100.0
+    return DigestSlot(
+        label=label,
+        statistic="rate",
+        cells=[f"{pct:.0f}% blocked", top],
+        entity=top,
+        magnitude=pct,
+    )
+
+
+# ── Lede formatters ─────────────────────────────────────────────────────────
+
+def _lede_client_volume(slot: DigestSlot) -> str:
+    return (
+        f"{slot.entity} issued {slot.magnitude:.0f}% of queries, "
+        f"{_format_ratio_lede(slot.ratio)} its nearest peer."
+    )
+
+
+def _lede_domain_volume(slot: DigestSlot) -> str:
+    return (
+        f"{slot.entity} was queried {int(slot.magnitude)} times, "
+        f"{_format_ratio_lede(slot.ratio)} the next domain."
+    )
+
+
+def _lede_query_length(slot: DigestSlot) -> str:
+    # Lede leads with owner (entity); cell order leads with maxlen.
+    return (
+        f"{slot.entity} issued a {int(slot.magnitude)}-character query, "
+        f"{_format_ratio_lede(slot.ratio)} the median length."
+    )
+
+
+def _lede_nxdomain_rate(slot: DigestSlot) -> str:
+    return (
+        f"{slot.magnitude:.0f}% of queries failed with NXDOMAIN, "
+        f"led by {slot.entity}."
+    )
+
+
+def _lede_block_rate(slot: DigestSlot) -> str:
+    return (
+        f"{slot.magnitude:.0f}% of queries were blocked, "
+        f"led by {slot.entity}."
+    )
+
+
+_INSIGHT_FORMATTERS = {
+    "client-volume":  _lede_client_volume,
+    "domain-volume":  _lede_domain_volume,
+    "query-length":   _lede_query_length,
+    "nxdomain-rate":  _lede_nxdomain_rate,
+    "block-rate":     _lede_block_rate,
+}
+
+
+# ── Zone 1 extras ───────────────────────────────────────────────────────────
+
+def _zone1_extras(frame: pd.DataFrame) -> list[tuple[str, str]]:
+    """Two lines, brief-pinned: distinct clients + distinct domains."""
+    if frame.empty:
+        return [("clients", "0"), ("domains", "0")]
+    distinct_clients = (
+        int(frame["src"].nunique(dropna=True)) if "src" in frame.columns else 0
+    )
+    distinct_domains = (
+        int(frame["query"].nunique(dropna=True)) if "query" in frame.columns else 0
+    )
+    return [
+        ("clients", str(distinct_clients)),
+        ("domains", str(distinct_domains)),
+    ]
+
+
+# ── Public entry point ──────────────────────────────────────────────────────
+
+def summarize(frame: pd.DataFrame, feed: str) -> dict:
+    """Return the schema-specific body of a dns DigestCard.
+
+    ``feed`` is ``"zeek"`` or ``"pihole"`` — selects which feed-specific
+    slots populate vs. return a non-speaking slot (which the summariser
+    then filters out of ``fields``). The four shared slots populate (or
+    stay non-speaking) the same way on both feeds.
+    """
+    from loghunter.digest._stats import select_insights_and_fields
+
+    slots = [
+        _slot_client_volume(frame),
+        _slot_domain_volume(frame),
+        _slot_query_length(frame),
+        _slot_qtype_mix(frame, feed),
+        _slot_nxdomain_rate(frame, feed),
+        _slot_block_rate(frame, feed),
+    ]
+    insights, fields = select_insights_and_fields(slots, _INSIGHT_FORMATTERS)
+    return {
+        "zone1_extras": _zone1_extras(frame),
+        "insights": insights,
+        "fields": fields,
+    }

loghunter/digest/syslog.py

@@ -0,0 +1,269 @@
+"""syslog summariser — orient-before-the-hunt for fidelity-aware syslog.
+
+The thinnest digest card by design — three slots, no manufactured depth.
+A three-row syslog card beside a six-row dns card honestly reads as "syslog
+is simpler," which is true; flat-grammar selection keeps it scannable.
+
+Slots (fixed order):
+  - host-volume    — cliff over per-host line counts (feed-independent)
+  - program-volume — cliff over per-program line counts (feed-independent)
+  - error-rate     — rate of lines that are "errors"; KIND forks on feed
+
+Fidelity fork (DNS precedent):
+
+  - feed ``"syslog"`` (flat rsyslog): the normalized frame carries no
+    severity field (RFC 3164 PRI is stripped by the parser), so "error" is
+    a keyword-token heuristic over the message body. Kind definition like
+    dns's "rcode == 3", not a badness threshold — gated only by RATE_FLOOR.
+  - feed ``"zeek"`` (Zeek syslog.log): Zeek emits an explicit ``severity``
+    enum on every line, so "error" is the real RFC 5424 error set
+    ``{EMERG, ALERT, CRIT, ERR}``. No keyword guessing.
+
+The lede formatter for ``error-rate`` forks its wording on ``feed`` — the
+Zeek arm speaks in severity terms, the flat arm in token terms. The card
+itself carries no footer surface under the flat grammar; the feed-difference
+disclosure is implicit in the insight wording.
+
+Cliff machinery imported from conn so the cards cannot drift on gate /
+floor / display-cap behaviour. The rate statistic and its RATE_FLOOR
+constant live in ``loghunter.digest._stats`` — factored once three cards
+needed an identical copy (this one, dns, and cloudtrail).
+"""
+
+from __future__ import annotations
+
+import re
+
+import pandas as pd
+
+from loghunter.common.finding import DigestSlot
+from loghunter.digest._stats import RATE_FLOOR, _rate
+from loghunter.digest.conn import (
+    CLIFF_DISPLAY_CAP,  # noqa: F401 — re-exported for downstream symmetry
+    CLIFF_GATE,         # noqa: F401 — re-exported for downstream symmetry
+    POPULATION_FLOOR,
+    _cliff,
+    _format_ratio_cell,
+    _format_ratio_lede,
+)
+
+
+# ── Calibration constants ───────────────────────────────────────────────────
+
+# Kind-definition heuristic. The normalized syslog frame carries no severity
+# field — RFC 3164 PRI is stripped by the parser and discarded. This is plain
+# text matching against an error-indicating token list, sorted longest-first so
+# multi-word phrases survive alternation as the list grows.
+_ERROR_TOKENS = (
+    "out of memory",
+    "unreachable",
+    "segfault",
+    "critical",
+    "failure",
+    "refused",
+    "timeout",
+    "denied",
+    "failed",
+    "error",
+    "fatal",
+    "panic",
+    "oom",
+)
+
+# Start-boundary at the alternation, free suffix at the end. So "errors" matches
+# "error" (start-bounded), "oom-killer" matches "oom" (hyphen is non-word),
+# "out of memory" matches as a literal phrase, but "terror" does NOT match
+# "error" (no word boundary before "error" when preceded by a word char).
+_ERROR_RE = re.compile(
+    r"\b(?:" + "|".join(re.escape(t) for t in _ERROR_TOKENS) + r")",
+    re.IGNORECASE,
+)
+
+# Zeek-feed kind: real RFC 5424 error severities. Uppercase enum strings on
+# the wire ("EMERG", "ALERT", "CRIT", "ERR") — matched case-insensitively to
+# absorb mixed-case Zeek emissions without column-sniffing the case shape.
+_SEVERITY_ERROR_SET = frozenset({"EMERG", "ALERT", "CRIT", "ERR"})
+
+# ── Slot computers ──────────────────────────────────────────────────────────
+
+def _slot_host_volume(frame: pd.DataFrame) -> DigestSlot:
+    """host-volume — cliff over per-host line counts."""
+    label = "host-volume"
+    if frame.empty or "host" not in frame.columns:
+        return DigestSlot(label=label, statistic="cliff")
+    counts = frame["host"].value_counts(dropna=True).sort_values(ascending=False)
+    result = _cliff(counts)
+    if result is None:
+        return DigestSlot(label=label, statistic="cliff")
+    entity, magnitude, ratio = result
+    total = len(frame)
+    share_pct = (magnitude / total * 100.0) if total > 0 else 0.0
+    entity_str = str(entity)
+    return DigestSlot(
+        label=label,
+        statistic="cliff",
+        cells=[entity_str, f"{share_pct:.0f}%", _format_ratio_cell(ratio)],
+        entity=entity_str,
+        magnitude=share_pct,
+        ratio=ratio,
+    )
+
+
+def _slot_program_volume(frame: pd.DataFrame) -> DigestSlot:
+    """program-volume — cliff over per-program line counts."""
+    label = "program-volume"
+    if frame.empty or "program" not in frame.columns:
+        return DigestSlot(label=label, statistic="cliff")
+    counts = frame["program"].value_counts(dropna=True).sort_values(ascending=False)
+    result = _cliff(counts)
+    if result is None:
+        return DigestSlot(label=label, statistic="cliff")
+    entity, magnitude, ratio = result
+    entity_str = str(entity)
+    return DigestSlot(
+        label=label,
+        statistic="cliff",
+        cells=[entity_str, f"{int(magnitude)}", _format_ratio_cell(ratio)],
+        entity=entity_str,
+        magnitude=magnitude,
+        ratio=ratio,
+    )
+
+
+def _slot_error_rate(frame: pd.DataFrame, feed: str) -> DigestSlot:
+    """error-rate — fraction of lines that are "errors". Kind forks on feed.
+
+    feed ``"zeek"``  : kind = ``severity`` ∈ {EMERG, ALERT, CRIT, ERR}. The
+                       severity column may be absent on a malformed Zeek frame
+                       — slot dashes in that case. Present-but-zero error-set
+                       values flows through ``_rate`` and dashes via the
+                       shared RATE_FLOOR (matching the flat-feed convention —
+                       neither feed paints "0%").
+    feed ``"syslog"``: kind = message-text keyword match (``_ERROR_RE``).
+                       Matching is against the canonical ``message`` column
+                       only (header-stripped body), never ``raw`` — the
+                       unstripped line would let timestamps or hostnames
+                       accidentally trip tokens.
+
+    Kind definition, not badness threshold: the fraction is reported as a
+    plain fact, gated only by the shared RATE_FLOOR.
+    """
+    label = "error-rate"
+    if frame.empty or "host" not in frame.columns:
+        return DigestSlot(label=label, statistic="rate")
+
+    if feed == "zeek":
+        if "severity" not in frame.columns:
+            return DigestSlot(label=label, statistic="rate")
+        severity = frame["severity"].astype(str).str.upper()
+        kind_mask = severity.isin(_SEVERITY_ERROR_SET)
+    else:
+        if "message" not in frame.columns:
+            return DigestSlot(label=label, statistic="rate")
+        messages = frame["message"].astype(str)
+        kind_mask = messages.str.contains(_ERROR_RE, na=False)
+
+    result = _rate(kind_mask, frame["host"])
+    if result is None:
+        return DigestSlot(label=label, statistic="rate")
+    fraction, top = result
+    pct = fraction * 100.0
+    return DigestSlot(
+        label=label,
+        statistic="rate",
+        cells=[f"{pct:.0f}%", top],
+        entity=top,
+        magnitude=pct,
+    )
+
+
+# ── Lede formatters ─────────────────────────────────────────────────────────
+
+def _lede_host_volume(slot: DigestSlot, feed: str) -> str:
+    return (
+        f"{slot.entity} emitted {slot.magnitude:.0f}% of log lines, "
+        f"{_format_ratio_lede(slot.ratio)} the next host."
+    )
+
+
+def _lede_program_volume(slot: DigestSlot, feed: str) -> str:
+    return (
+        f"{slot.entity} emitted {int(slot.magnitude)} lines, "
+        f"{_format_ratio_lede(slot.ratio)} the next program."
+    )
+
+
+def _lede_error_rate(slot: DigestSlot, feed: str) -> str:
+    """error-rate lede — wording forks on feed.
+
+    The Zeek variant MUST NOT say "token" or imply keyword matching — that
+    would misdescribe the real-severity Zeek path. The flat-syslog variant
+    keeps the existing keyword wording.
+    """
+    if feed == "zeek":
+        return (
+            f"{slot.magnitude:.0f}% of lines are error-severity "
+            f"(ERR or higher), led by {slot.entity}."
+        )
+    return (
+        f"{slot.magnitude:.0f}% of lines carry an error token, "
+        f"led by {slot.entity}."
+    )
+
+
+def _insight_formatters(feed: str) -> dict[str, "Callable[[DigestSlot], str]"]:
+    """Bind ``feed`` into the feed-aware formatters so the shared selection
+    helper sees the standard ``(slot) -> str`` shape.
+
+    A small dedicated helper rather than a sentinel — the formatters take
+    feed explicitly, and partial-binding here is the obvious mechanism.
+    """
+    return {
+        "host-volume":    lambda slot: _lede_host_volume(slot, feed),
+        "program-volume": lambda slot: _lede_program_volume(slot, feed),
+        "error-rate":     lambda slot: _lede_error_rate(slot, feed),
+    }
+
+
+# ── Zone 1 extras ───────────────────────────────────────────────────────────
+
+def _zone1_extras(frame: pd.DataFrame) -> list[tuple[str, str]]:
+    """Two lines, brief-pinned: distinct hosts + distinct programs."""
+    if frame.empty:
+        return [("hosts", "0"), ("programs", "0")]
+    distinct_hosts = (
+        int(frame["host"].nunique(dropna=True)) if "host" in frame.columns else 0
+    )
+    distinct_programs = (
+        int(frame["program"].nunique(dropna=True)) if "program" in frame.columns else 0
+    )
+    return [
+        ("hosts", str(distinct_hosts)),
+        ("programs", str(distinct_programs)),
+    ]
+
+
+# ── Public entry point ─────────────────────────────────────────────────────
+
+def summarize(frame: pd.DataFrame, feed: str) -> dict:
+    """Return the schema-specific body of a syslog DigestCard.
+
+    ``feed`` is ``"zeek"`` or ``"syslog"`` — picks which kind drives the
+    error-rate slot and which wording the lede uses. Host- and program-volume
+    cliffs are feed-independent.
+    """
+    from loghunter.digest._stats import select_insights_and_fields
+
+    slots = [
+        _slot_host_volume(frame),
+        _slot_program_volume(frame),
+        _slot_error_rate(frame, feed),
+    ]
+    insights, fields = select_insights_and_fields(
+        slots, _insight_formatters(feed),
+    )
+    return {
+        "zone1_extras": _zone1_extras(frame),
+        "insights": insights,
+        "fields": fields,
+    }