tinybeeai 0.0.0-stage → 0.16.1

package/dist/updates/index.js

@@ -0,0 +1,835 @@
+// The dot reports like an async teammate (KAT-143). Live events become drafts (sources.ts), the router
+// sets a level (route.ts), and this module keeps the feed and delivers:
+//
+//   now     the dot's web chat, its Slack channel or DM, and the phone, at once
+//   thread  a reply where someone asked: their chat turn or their Slack thread
+//   digest  held for the next digest (every few hours) or the morning note
+//   drop    nothing; a passing run nobody asked about is never a message
+//
+// Every update has one key. The feed row is written before anything is sent, under a unique index, so
+// a repeated event or a second source for the same thing (the probe and the outage run) sends nothing
+// twice on any channel. Quiet hours hold the phone and Slack DMs; the morning note lists what they held.
+//
+// Slack and the phone can fail (a Slack error, a push error, the network). Every try is recorded on the
+// feed row. An urgent update (level now, not a digest or morning note) retries a failed channel after
+// 1, 5, and 15 minutes, then every 15 minutes until it goes through (KAT-271). From the fourth failed try
+// the failure also shows on the dot page and in the Inbox, and the retries go on. The retry is written to
+// the row before the first try of any channel, so a restart in the middle of a send still finds it.
+// A channel never has two tries in flight, so a slow first try that still succeeds is never followed by a
+// second send, and a channel that went through is never tried again. Digests, morning notes, and thread
+// replies stay one-shot.
+//
+// A listed failure can be cleared by a person: Retry sends it again now on its failed channels, and Dismiss
+// takes it off the list without sending and ends its retries (retryNow, dismissFailed, dismissAllFailed).
+import { redactCode } from '../agent/redact.js';
+import { log } from '../config.js';
+import { sandboxOn, sandboxSkips } from '../sandbox.js';
+import { count, get, insert, list, update } from '../db.js';
+import { publish, subscribe } from '../events.js';
+import { notify } from '../notify/push.js';
+import { recordDelivery } from '../monitor/self-health.js';
+import { postUpdate } from '../slack/index.js';
+import { ruleFor } from '../agent/rules.js';
+import { liveCount } from '../agent/approvals.js';
+import { renderRefs } from '../refs.js';
+import { openLinks } from '../link-trust.js';
+import { digestRuns } from '../flows/digest.js';
+import { morningText, reviewItem, TEAM_DETAILS_HEADING } from './words.js';
+import { askForJob, askForRun } from './asks.js';
+import { setPushAdopter, setUpdatesOwnDelivery } from './owner.js';
+import { inQuietHours, ME, morningMinute, personWants, prefsFor } from './prefs.js';
+import { routeUpdate } from './route.js';
+import { updatesFromEvent } from './sources.js';
+export { setUpdateDecider, ruleLevel, routeUpdate } from './route.js';
+export { updatesFromEvent } from './sources.js';
+const defaultDeps = {
+    now: () => new Date(),
+    slack: postUpdate,
+    push: async (n) => {
+        const r = await notify(n);
+        if (r.failed)
+            throw new Error(r.failed);
+        return !r.deduped;
+    },
+    attemptMs: 20_000,
+};
+/** Minutes to wait after the 1st, 2nd, and 3rd failed try of an urgent update. Later failures wait as long as the last. */
+export const RETRY_MINUTES = [1, 5, 15];
+/** Tries kept on a row per channel; older ones drop off so a long outage does not grow the row without end. */
+const KEEP_ATTEMPTS = 10;
+/** A try that has not answered after this long no longer blocks a retry (a hung connection). */
+const HUNG_MS = 5 * 60_000;
+let deps = defaultDeps;
+export function setUpdateDeps(d) {
+    deps = d ? { ...defaultDeps, ...d } : defaultDeps;
+}
+const digestHours = () => {
+    const n = Number(process.env.LOCALDOST_UPDATE_DIGEST_HOURS);
+    return Number.isFinite(n) && n > 0 ? n : 3;
+};
+// ---------- feed ----------
+export function feedFor(watchId, limit = 50) {
+    return list('updates', { watchId }, { orderBy: 'id DESC', limit });
+}
+function byKey(key) {
+    return list('updates', { key }, { limit: 1 })[0];
+}
+function patchUpdate(u, patch) {
+    update('updates', u.id, patch);
+    return { ...u, ...patch };
+}
+function context() {
+    return {
+        askForRun: (run) => askForRun(run),
+        askForJob: (id) => askForJob(id),
+        downSince: (watchId) => {
+            const rows = list('probe_state', { watchId }).filter((r) => r.down && r.downSince);
+            return rows.map((r) => r.downSince).sort()[0];
+        },
+        has: (key) => Boolean(byKey(key)),
+        dotName: (watchId) => get('watches', watchId)?.name ?? 'Your app',
+        isLead: (watchId) => get('watches', watchId)?.kind === 'lead',
+    };
+}
+// ---------- text ----------
+/** Markdown to one plain line for a phone: link labels stay, link syntax and emphasis go. */
+export function plainLine(md, max = 160) {
+    const t = md
+        .replace(/\[([^\]]+)\]\([^)\s]+\)/g, '$1')
+        .replace(/[*_`]/g, '')
+        .replace(/\s+/g, ' ')
+        .trim();
+    return t.length > max ? `${t.slice(0, max - 3).trimEnd()}...` : t;
+}
+function firstLine(md) {
+    return md.split('\n').find((l) => l.trim()) ?? '';
+}
+// ---------- delivery ----------
+function slackTarget(w) {
+    const mode = w.updates?.slack ?? 'channel';
+    if (mode === 'off')
+        return null;
+    if (mode === 'dm') {
+        const u = w.updates?.slackUser?.trim();
+        return u ? { channel: u, person: `slack:${u}` } : null;
+    }
+    return w.slackChannel ? { channel: w.slackChannel.replace(/^#/, '') } : null;
+}
+function category(u) {
+    if (u.kind === 'decision')
+        return 'drafts';
+    if (u.kind === 'prod_down' || u.kind === 'prod_cause' || u.kind === 'prod_up')
+        return 'outages';
+    if (u.kind === 'bug' || u.kind === 'defect')
+        return 'bugs';
+    return undefined;
+}
+/** What follows "## Details" in the saved verdict of a team digest run: each member's own digest and the counts. Empty when there is none. */
+function teamDigestDetails(runId) {
+    const summary = get('runs', runId)?.verdict?.summary ?? '';
+    const at = summary.indexOf(TEAM_DETAILS_HEADING);
+    return at < 0 ? '' : summary.slice(at + TEAM_DETAILS_HEADING.length).trim();
+}
+/** Posts the update to the dot's web chat as the dot, and tells open apps. */
+async function toChat(w, u, extra = {}) {
+    const askedLine = u.origin?.channel === 'web' && u.origin.text ? `You asked: "${plainLine(u.origin.text, 80)}"\n\n` : '';
+    // Asked in the all-dots chat: the reply goes there, with the dot's name since that chat has many dots.
+    const inAll = u.origin?.channel === 'web' && u.origin.scope === 'all';
+    // Asked in a lead's chat (KAT-405): a member did the work, and the answer lands in the lead's chat with the member's name first.
+    const leadId = u.origin?.channel === 'web' && u.origin.scope === 'lead' ? u.origin.leadId : undefined;
+    const fromLine = inAll || leadId ? `From ${w.name}\n\n` : '';
+    const content = await renderRefs(openLinks(`${askedLine}${fromLine}**${u.title}**\n${u.body}`), { dotId: w.id, format: 'markdown' });
+    // The team digest of a lead keeps each member's own digest and the counts under a Details fold in the chat (KAT-405). Slack gets the body alone.
+    // The members' digests carry titles from PRs and tickets, so a link the server did not build shows its address there too (KAT-380).
+    const details = w.kind === 'lead' && u.kind === 'daily_digest' && u.runId ? teamDigestDetails(u.runId) : '';
+    const detailsText = details ? await renderRefs(openLinks(details), { dotId: w.id, format: 'markdown' }) : '';
+    const meta = {
+        id: u.id,
+        kind: u.kind,
+        level: u.level,
+        ...(u.severity ? { severity: u.severity } : {}),
+        ...(u.runId ? { runId: u.runId } : {}),
+        ...(u.approvalId ? { approvalId: u.approvalId } : {}),
+        ...(u.url ? { url: u.url } : {}),
+        ...(extra.replyTo ? { replyTo: extra.replyTo } : {}),
+        ...(extra.approval ? { approval: extra.approval } : {}),
+        ...(detailsText ? { details: detailsText } : {}),
+    };
+    const row = insert('messages', { watchId: inAll ? 'all' : (leadId ?? w.id), role: 'assistant', content, kind: 'update', update: meta });
+    publish({ type: 'update', update: u, message: row });
+    return row;
+}
+/** A digest or a morning note: level now, but one quiet message that never retries. */
+function isNote(kind) {
+    return kind === 'digest' || kind === 'morning' || kind === 'daily_digest';
+}
+function noticeOf(u, urgency) {
+    const cat = category(u);
+    return {
+        key: u.key,
+        title: plainLine(u.title, 80),
+        body: plainLine(firstLine(u.body), 160),
+        ...(u.url ? { url: u.url } : {}),
+        urgency,
+        ...(cat ? { category: cat } : {}),
+        watchId: u.watchId,
+    };
+}
+function plan(w, u, d) {
+    const urgent = u.severity === 'critical' || u.severity === 'high';
+    const base = {
+        title: u.title,
+        body: u.body,
+        key: u.key,
+        ...(u.runId ? { runId: u.runId } : {}),
+        ...(u.url ? { url: u.url } : {}),
+        ...(u.linkLabel ? { linkLabel: u.linkLabel } : {}),
+    };
+    if (u.level === 'thread') {
+        const o = u.origin;
+        if (o?.channel === 'slack') {
+            return {
+                chat: !d.passing,
+                slack: { post: { ...base, channel: o.slackChannel, threadTs: o.threadTs, asked: true, ...(u.approvalId ? { approvalId: u.approvalId } : {}) } },
+                push: null,
+                needsThem: false,
+            };
+        }
+        if (o?.channel === 'web') {
+            // The asking turn already shows the run card as it runs; a passing result needs no second message.
+            return { chat: !d.passing, slack: null, push: urgent ? noticeOf(u, 'high') : null, needsThem: urgent };
+        }
+        // A follow-up (prod_cause): under its parent in Slack, and in the chat.
+        const parent = u.parentKey ? byKey(u.parentKey) : undefined;
+        const target = slackTarget(w);
+        const ts = parent?.slackTs;
+        const channel = parent?.slackChannel ?? target?.channel;
+        return {
+            chat: true,
+            slack: ts && channel ? { post: { ...base, channel, threadTs: ts } } : null,
+            push: null,
+            needsThem: false,
+        };
+    }
+    if (u.level === 'now') {
+        const target = slackTarget(w);
+        const urgency = isNote(u.kind) ? 'low' : urgent ? 'high' : 'normal';
+        // A push a flow asked for (KAT-281): phone only. The chat and Slack have their own messages for the run.
+        if (u.kind === 'push')
+            return { chat: false, slack: null, push: noticeOf(u, u.urgency ?? 'normal'), needsThem: true };
+        return {
+            chat: true,
+            slack: target
+                ? {
+                    post: { ...base, channel: target.channel, ...(u.approvalId ? { approvalId: u.approvalId } : {}) },
+                    ...(target.person ? { person: target.person } : {}),
+                }
+                : null,
+            push: noticeOf(u, urgency),
+            needsThem: u.kind !== 'digest' && u.kind !== 'daily_digest',
+        };
+    }
+    return { chat: false, slack: null, push: null, needsThem: false };
+}
+// ---------- tries, retries ----------
+/** Urgent: worth retrying. Digests and morning notes (level now, but a note) and thread replies are one-shot. */
+function retryable(u) {
+    return u.level === 'now' && !isNote(u.kind);
+}
+/** Tries in flight, `<update id>:<channel>` to the start time. One per channel, so a retry never overlaps a slow try. */
+const inflight = new Map();
+function busy(id, channel) {
+    const at = inflight.get(`${id}:${channel}`);
+    return at !== undefined && Date.now() - at < HUNG_MS;
+}
+function errorText(err) {
+    const m = err instanceof Error ? err.message : String(err);
+    return m.replace(/\s+/g, ' ').trim().slice(0, 200) || 'unknown error';
+}
+/** Reads the row fresh and writes the change, with no await in between, so late results never overwrite each other. */
+function mutate(id, fn) {
+    const u = get('updates', id);
+    if (!u)
+        return undefined;
+    return patchUpdate(u, fn(u));
+}
+function lastAttempt(u, channel) {
+    return [...(u.attempts ?? [])].reverse().find((a) => a.channel === channel);
+}
+/** When the next retry of a channel is due: after its last failed try, by how many tries it had. */
+function dueAt(u, channel) {
+    const last = lastAttempt(u, channel);
+    // No try on record yet: the server stopped in the middle of the first one; the written due time stands.
+    if (!last)
+        return u.retryAt ? Date.parse(u.retryAt) : undefined;
+    if (last.ok)
+        return undefined;
+    const wait = RETRY_MINUTES[Math.min(last.n, RETRY_MINUTES.length) - 1] ?? RETRY_MINUTES[RETRY_MINUTES.length - 1];
+    return Date.parse(last.at) + wait * 60_000;
+}
+/** The retry fields for a set of waiting channels, kept in step with the indexed flags. */
+function retryFields(u, retrying, failed) {
+    const dues = [...retrying].map((c) => dueAt(u, c)).filter((t) => t !== undefined);
+    return {
+        retrying: [...retrying],
+        retryPending: retrying.size > 0,
+        retryAt: dues.length ? new Date(Math.min(...dues)).toISOString() : undefined,
+        failed: [...failed],
+        failedDelivery: failed.size > 0,
+    };
+}
+/**
+ * Runs one try on a channel and settles it within attemptMs. A try that does not answer in time counts as
+ * failed (the retry clock starts), but stays in flight: the retry waits for it, and if it finally succeeds
+ * the row is corrected, so a slow first try and its retry never both send.
+ */
+async function tryChannel(id, channel, send) {
+    const k = `${id}:${channel}`;
+    inflight.set(k, Date.now());
+    const call = (async () => send())().then((r) => ({ ok: true, r }), (err) => ({ ok: false, err }));
+    let timer;
+    const late = new Promise((resolve) => {
+        timer = setTimeout(() => resolve('late'), deps.attemptMs);
+        timer.unref?.();
+    });
+    const first = await Promise.race([call, late]);
+    clearTimeout(timer);
+    if (first === 'late') {
+        void call.then((res) => {
+            inflight.delete(k);
+            if (res.ok && res.r.sent)
+                lateSuccess(id, channel, res.r.patch);
+            else if (!res.ok)
+                log.warn(`updates: ${channel} try for "${id}" failed after the wait: ${errorText(res.err)}`);
+        });
+        return { outcome: 'failed', error: `no answer after ${Math.max(1, Math.round(deps.attemptMs / 1000))} seconds` };
+    }
+    inflight.delete(k);
+    if (!first.ok)
+        return { outcome: 'failed', error: errorText(first.err) };
+    return { outcome: first.r.sent ? 'sent' : 'skipped', ...(first.r.patch ? { patch: first.r.patch } : {}) };
+}
+/** A try that was counted as failed for being slow went through after all: record it, drop the retry. */
+function lateSuccess(id, channel, patch) {
+    const row = mutate(id, (u) => {
+        const attempts = [...(u.attempts ?? [])];
+        for (let i = attempts.length - 1; i >= 0; i--) {
+            const a = attempts[i];
+            if (a.channel !== channel)
+                continue;
+            const { error: _error, ...rest } = a;
+            attempts[i] = { ...rest, ok: true };
+            break;
+        }
+        const next = { ...u, attempts };
+        const retrying = new Set(u.retrying ?? []);
+        const failed = new Set(u.failed ?? []);
+        retrying.delete(channel);
+        failed.delete(channel);
+        return { ...patch, attempts, delivered: [...new Set([...u.delivered, channel])], ...retryFields(next, retrying, failed) };
+    });
+    if (row) {
+        publish({ type: 'update', update: row });
+        recordDelivery(row.watchId, channel, true);
+    }
+}
+/** Records a try on the row and sets what happens next for an urgent update. Returns the fresh row. */
+function applyTry(id, channel, t, at) {
+    const row = mutate(id, (u) => {
+        let attempts = [...(u.attempts ?? [])];
+        const n = (lastAttempt(u, channel)?.n ?? 0) + 1;
+        if (t.outcome !== 'skipped')
+            attempts.push({ channel, n, at: at.toISOString(), ok: t.outcome === 'sent', ...(t.error ? { error: t.error } : {}) });
+        // Keep the newest tries of this channel; `n` keeps counting, so the backoff does not start over.
+        const own = attempts.filter((a) => a.channel === channel);
+        if (own.length > KEEP_ATTEMPTS) {
+            const drop = new Set(own.slice(0, own.length - KEEP_ATTEMPTS));
+            attempts = attempts.filter((a) => !drop.has(a));
+        }
+        const next = { ...u, attempts };
+        const retrying = new Set(u.retrying ?? []);
+        const failed = new Set(u.failed ?? []);
+        retrying.delete(channel);
+        const patch = { attempts };
+        if (t.outcome === 'sent') {
+            failed.delete(channel);
+            patch.delivered = [...new Set([...u.delivered, channel])];
+            Object.assign(patch, t.patch);
+        }
+        else if (t.outcome === 'failed' && retryable(u)) {
+            // Retries go on until the channel delivers once (KAT-271). From the 4th failed try the failure is also listed.
+            retrying.add(channel);
+            if (n > RETRY_MINUTES.length)
+                failed.add(channel);
+        }
+        return { ...patch, ...retryFields(next, retrying, failed) };
+    });
+    if (row && t.outcome === 'failed') {
+        if (row.failed?.includes(channel))
+            log.warn(`updates: "${row.key}" still not sent on ${channel}, trying again later: ${t.error}`);
+        publish({ type: 'update', update: row });
+    }
+    // Self-health: the same error on a channel three tries in a row raises an outage on the dot (dogfood ledger).
+    if (row && t.outcome !== 'skipped')
+        recordDelivery(row.watchId, channel, t.outcome === 'sent', t.error, at);
+    return row;
+}
+function gateFor(w, u, p, channel, needsThem, now) {
+    const note = isNote(u.kind);
+    const personal = (person) => {
+        const prefs = prefsFor(person);
+        // A digest or morning note counts as digest for the person's level, though it goes out at once.
+        const as = note ? 'digest' : u.level === 'thread' ? 'now' : u.level;
+        if (!personWants(prefs, as, { needsThem }))
+            return 'skip';
+        return inQuietHours(prefs, now) ? 'hold' : 'send';
+    };
+    if (channel === 'slack') {
+        if (!p.slack)
+            return 'skip';
+        return p.slack.person ? personal(p.slack.person) : 'send';
+    }
+    // The phone follows the push_notify rule, the person's level, and quiet hours.
+    if (!p.push || ruleFor(w, 'push_notify') !== 'act')
+        return 'skip';
+    // One phone alert per run outcome: a push a flow asked for and the router's own update for the same run never both go out.
+    if (u.kind !== 'push' && OUTCOME_KINDS.has(u.kind) && phoneAlreadyTold(u))
+        return 'skip';
+    return personal(ME);
+}
+/** Update kinds that report how a run ended. A flow's own push about the same run covers them on the phone. */
+const OUTCOME_KINDS = new Set(['bug', 'defect', 'result', 'prod_down', 'prod_up', 'daily_digest']);
+/** True when the phone has this update's alert, is about to get it, or quiet hours hold it. */
+function phoneStanding(u) {
+    return u.delivered.includes('push') || Boolean(u.retrying?.includes('push')) || Boolean(u.held?.includes('push'));
+}
+/** Whether another update of the same run, of the other family (a flow's push or a run outcome), already has the phone. */
+function phoneAlreadyTold(u) {
+    if (!u.runId)
+        return false;
+    const mine = u.kind === 'push';
+    return list('updates', { watchId: u.watchId, runId: u.runId }).some((r) => r.id !== u.id && (mine ? OUTCOME_KINDS.has(r.kind) : r.kind === 'push') && phoneStanding(r));
+}
+function senderFor(w, u, p, channel) {
+    if (channel === 'slack') {
+        const post = p.slack.post;
+        return async () => {
+            const res = await deps.slack(w, post);
+            if (res.ok)
+                return { sent: true, patch: { slackChannel: res.channel, ...(res.ts ? { slackTs: res.ts } : {}) } };
+            log.info(`updates: Slack post "${u.key}" not sent: ${res.reason}`);
+            return { sent: false };
+        };
+    }
+    const notice = p.push;
+    return async () => ({ sent: await deps.push(notice) });
+}
+async function deliver(w, u, d) {
+    const p = plan(w, u, d);
+    const now = deps.now();
+    const note = isNote(u.kind);
+    const needsThem = p.needsThem || (note && Boolean(d.needsThem));
+    const urgent = retryable(u);
+    const held = [];
+    // Decide every channel and write the retry before anything is awaited (KAT-271). A restart in the middle of
+    // the first try of one channel then still leaves the other channel, and this one, waiting for a retry.
+    const gates = { slack: gateFor(w, u, p, 'slack', needsThem, now), push: gateFor(w, u, p, 'push', needsThem, now) };
+    if (urgent) {
+        const waiting = ['slack', 'push'].filter((c) => gates[c] === 'send');
+        if (waiting.length)
+            mutate(u.id, (r) => {
+                const at = new Date(now.getTime() + deps.attemptMs + RETRY_MINUTES[0] * 60_000).toISOString();
+                return { retrying: [...new Set([...(r.retrying ?? []), ...waiting])], retryPending: true, retryAt: r.retryAt && r.retryAt < at ? r.retryAt : at };
+            });
+    }
+    if (p.chat) {
+        try {
+            const replyTo = u.origin?.channel === 'web' ? u.origin.messageId : undefined;
+            await toChat(w, u, { ...(replyTo ? { replyTo } : {}), ...(d.approval ? { approval: d.approval } : {}) });
+            mutate(u.id, (r) => ({ delivered: [...new Set([...r.delivered, 'chat'])] }));
+        }
+        catch (err) {
+            log.warn(`updates: chat post failed: ${String(err)}`);
+        }
+    }
+    for (const channel of ['slack', 'push']) {
+        const gate = gates[channel];
+        if (gate === 'hold') {
+            held.push(channel);
+            continue;
+        }
+        if (gate !== 'send')
+            continue;
+        const t = await tryChannel(u.id, channel, senderFor(w, u, p, channel));
+        if (t.outcome === 'failed')
+            log.warn(`updates: ${channel} for "${u.key}" failed: ${t.error}`);
+        applyTry(u.id, channel, t, now);
+    }
+    const fresh = held.length ? mutate(u.id, () => ({ held })) : get('updates', u.id);
+    return fresh ?? u;
+}
+/**
+ * Sends again what failed on an urgent update: each waiting channel whose backoff has passed, unless a
+ * try is still in flight on it. The plan is rebuilt from the row, and the channel's gate is asked again, so
+ * quiet hours that began since hold the retry for the morning note and a person who turned the phone off
+ * hears nothing. Returns the number of retries that went through.
+ */
+export async function retryRound(now = deps.now()) {
+    let sent = 0;
+    for (const row of list('updates', { retryPending: true }, { orderBy: 'id ASC', limit: 100 })) {
+        const w = get('watches', row.watchId);
+        if (!w || w.archived) {
+            mutate(row.id, (u) => retryFields(u, new Set(), new Set(u.failed ?? [])));
+            continue;
+        }
+        // A decision that was approved, rejected, or expired needs no alert any more.
+        if (row.kind === 'decision' && row.approvalId && get('approvals', row.approvalId)?.status !== 'pending') {
+            mutate(row.id, (u) => retryFields(u, new Set(), new Set()));
+            continue;
+        }
+        for (const channel of row.retrying ?? []) {
+            const u = get('updates', row.id);
+            if (!u || !u.retrying?.includes(channel))
+                continue;
+            if (u.delivered.includes(channel)) {
+                mutate(u.id, (r) => retryFields(r, new Set((r.retrying ?? []).filter((c) => c !== channel)), new Set(r.failed ?? [])));
+                continue;
+            }
+            const due = dueAt(u, channel);
+            if (due === undefined || due > now.getTime() || busy(u.id, channel))
+                continue;
+            const p = plan(w, u, {});
+            const gate = gateFor(w, u, p, channel, p.needsThem, now);
+            if (gate !== 'send') {
+                mutate(u.id, (r) => ({
+                    ...(gate === 'hold' ? { held: [...new Set([...(r.held ?? []), channel])] } : {}),
+                    ...retryFields(r, new Set((r.retrying ?? []).filter((c) => c !== channel)), new Set(r.failed ?? [])),
+                }));
+                continue;
+            }
+            const t = await tryChannel(u.id, channel, senderFor(w, u, p, channel));
+            if (t.outcome === 'failed')
+                log.warn(`updates: retry on ${channel} for "${u.key}" failed: ${t.error}`);
+            applyTry(u.id, channel, t, now);
+            if (t.outcome === 'sent')
+                sent++;
+        }
+    }
+    return sent;
+}
+/** Urgent updates that failed on a channel at least three retries in a row, newest first. Shown on the dot page and in the Inbox. */
+export function failedDeliveries(watchId, limit = 50) {
+    return list('updates', { failedDelivery: true, ...(watchId ? { watchId } : {}) }, { orderBy: 'id DESC', limit });
+}
+/** How many alerts are in the failed list, for all dots or one. `failedDeliveries` returns at most a page of them. */
+export function countFailedDeliveries(watchId) {
+    return count('updates', { failedDelivery: true, ...(watchId ? { watchId } : {}) });
+}
+/** Why a failed alert cannot be sent or dismissed. `code` maps to the HTTP status in the route. */
+export class FailedAlertError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+    }
+}
+/** True when the update is a decision whose approval is no longer pending (approved, rejected, expired, superseded). */
+export function approvalSettled(u) {
+    return u.kind === 'decision' && Boolean(u.approvalId) && get('approvals', u.approvalId)?.status !== 'pending';
+}
+/**
+ * Sends a failed urgent update again now (the Retry button), on its failed channels only. It uses the same
+ * plan, sender, and try as the timed retries, so the attempts history grows and a success corrects the row.
+ * A person asked, so quiet hours do not hold it. A channel the person turned off, or that has no target
+ * any more, is not sent and is reported. A decision whose approval was settled is refused: nobody needs it.
+ */
+export async function retryNow(id, now = deps.now()) {
+    const row = get('updates', id);
+    if (!row)
+        throw new FailedAlertError('not_found', 'update not found');
+    if (!row.failedDelivery || !row.failed?.length)
+        throw new FailedAlertError('not_failed', 'This alert is not in the failed list.');
+    const w = get('watches', row.watchId);
+    if (!w || w.archived)
+        throw new FailedAlertError('archived', 'The dot is archived. Restore it to send this alert.');
+    if (approvalSettled(row))
+        throw new FailedAlertError('settled', 'The approval is already settled, so this alert is not worth sending.');
+    const sent = [];
+    const failed = [];
+    for (const channel of [...row.failed]) {
+        const u = get('updates', id);
+        if (!u || u.delivered.includes(channel))
+            continue;
+        if (busy(id, channel)) {
+            failed.push({ channel, error: 'A try is still running. Wait a moment and retry.' });
+            continue;
+        }
+        const p = plan(w, u, {});
+        if (gateFor(w, u, p, channel, p.needsThem, now) === 'skip') {
+            failed.push({ channel, error: `${channel === 'slack' ? 'Slack' : 'The phone'} is turned off for this dot, so nothing was sent.` });
+            continue;
+        }
+        const t = await tryChannel(id, channel, senderFor(w, u, p, channel));
+        if (t.outcome === 'failed')
+            log.warn(`updates: manual retry on ${channel} for "${u.key}" failed: ${t.error}`);
+        applyTry(id, channel, t, now);
+        if (t.outcome === 'sent')
+            sent.push(channel);
+        else
+            failed.push({ channel, error: t.error ?? 'The send was skipped.' });
+    }
+    const fresh = get('updates', id) ?? row;
+    publish({ type: 'update', update: fresh });
+    return { update: fresh, sent, failed, cleared: !fresh.failedDelivery };
+}
+/** The fields that end a failed alert's retries and take it off the failed list. */
+function dismissPatch(by, at) {
+    return {
+        failed: [],
+        failedDelivery: false,
+        retrying: [],
+        retryPending: false,
+        retryAt: undefined,
+        dismissedBy: by,
+        dismissedAt: at.toISOString(),
+    };
+}
+/**
+ * Takes one failed alert off the failed list without sending (the Dismiss button). The retries stop too,
+ * or the next failed try would list it again. The feed row, the chat message, and the attempts stay.
+ */
+export function dismissFailed(id, by, now = deps.now()) {
+    const row = get('updates', id);
+    if (!row)
+        throw new FailedAlertError('not_found', 'update not found');
+    if (!row.failedDelivery)
+        throw new FailedAlertError('not_failed', 'This alert is not in the failed list.');
+    const next = patchUpdate(row, dismissPatch(by, now));
+    publish({ type: 'update', update: next });
+    return next;
+}
+/** Dismisses every failed alert, or those of one dot, or those created before `olderThan`. Returns the ones it cleared. */
+export function dismissAllFailed(by, opts = {}, now = deps.now()) {
+    const before = opts.olderThan ? Date.parse(opts.olderThan) : undefined;
+    const rows = failedDeliveries(opts.dotId, 100_000).filter((u) => before === undefined || Date.parse(u.createdAt) < before);
+    return rows.map((u) => {
+        const next = patchUpdate(u, dismissPatch(by, now));
+        publish({ type: 'update', update: next });
+        return next;
+    });
+}
+/** Stores and delivers one draft. Returns the feed row, or null for a repeat or a dropped draft. */
+export async function handleDraft(d, decision) {
+    if (byKey(d.key))
+        return null;
+    const w = get('watches', d.watchId);
+    if (!w || w.archived)
+        return null;
+    const route = decision ?? (await routeUpdate(d));
+    if (route.level === 'drop')
+        return null;
+    let row;
+    try {
+        row = insert('updates', {
+            watchId: d.watchId,
+            kind: d.kind,
+            level: route.level,
+            key: d.key,
+            // Run errors can carry tokens or passwords: the stored text is masked once, so chat, Slack, and push
+            // all send the masked version.
+            title: redactCode(d.title),
+            body: redactCode(d.body),
+            reason: route.reason,
+            delivered: [],
+            ...(d.url ? { url: d.url } : {}),
+            ...(d.linkLabel ? { linkLabel: d.linkLabel } : {}),
+            ...(d.severity ? { severity: d.severity } : {}),
+            ...(d.reproduced !== undefined ? { reproduced: d.reproduced } : {}),
+            ...(d.urgency ? { urgency: d.urgency } : {}),
+            ...(d.runId ? { runId: d.runId } : {}),
+            ...(d.approvalId ? { approvalId: d.approvalId } : {}),
+            ...(d.jobId ? { jobId: d.jobId } : {}),
+            ...(d.origin ? { origin: d.origin } : {}),
+            ...(d.parentKey ? { parentKey: d.parentKey } : {}),
+        });
+    }
+    catch {
+        // The unique key: another event for the same thing got there first.
+        return null;
+    }
+    if (route.level === 'digest')
+        return row;
+    return deliver(w, row, d);
+}
+/** Drafts for one event, handled in order. */
+export async function handleEvent(e) {
+    if (e.type === 'update')
+        return [];
+    const drafts = updatesFromEvent(e, context(), (id) => get('runs', id));
+    const out = [];
+    for (const d of drafts) {
+        const r = await handleDraft(d);
+        if (r)
+            out.push(r);
+    }
+    return out;
+}
+// ---------- digests and the morning note ----------
+/** How long after the due minute the morning note may still go out. */
+const MORNING_WINDOW_MIN = 6 * 60;
+function stateOf(watchId) {
+    return get('update_state', watchId) ?? insert('update_state', { id: watchId });
+}
+function localDay(d) {
+    return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`;
+}
+/** Feed rows waiting for a digest: digest-level ones, plus urgent ones quiet hours held from the phone. */
+export function pendingFor(watchId) {
+    return list('updates', { watchId, digestedAt: null }, { orderBy: 'id ASC', limit: 500 }).filter((u) => u.level === 'digest' || (u.held?.length ?? 0) > 0);
+}
+/** The digest lines for a set of feed rows: one line per update with its link, then one count of the checks that started. */
+export function digestLines(rows) {
+    const picked = rows.filter((u) => u.kind === 'picked_up').length;
+    const lines = rows
+        .filter((u) => u.kind !== 'picked_up')
+        .map((u) => `- ${plainLine(u.title, 120)}${u.url ? ` [${u.linkLabel ?? 'Open details'}](${u.url})` : ''}`);
+    if (picked)
+        lines.push(`- Started ${picked === 1 ? '1 check' : `${picked} checks`}.`);
+    return lines;
+}
+async function sendNote(w, kind, key, title, body, needsThem, rows) {
+    const out = await handleDraft({ watchId: w.id, kind, key, title, body, url: `/#/chat/${encodeURIComponent(w.id)}`, needsThem });
+    const at = deps.now().toISOString();
+    for (const r of rows)
+        patchUpdate(r, { digestedAt: at });
+    return out;
+}
+/** A digest round for one dot when one is due and something waits. */
+export async function digestRound(w, now = deps.now()) {
+    const st = stateOf(w.id);
+    if (!st.lastDigestAt) {
+        update('update_state', w.id, { lastDigestAt: now.toISOString() });
+        return null;
+    }
+    if (now.getTime() - Date.parse(st.lastDigestAt) < digestHours() * 3600_000)
+        return null;
+    if (inQuietHours(prefsFor(ME), now))
+        return null;
+    const rows = pendingFor(w.id).filter((u) => u.level === 'digest');
+    update('update_state', w.id, { lastDigestAt: now.toISOString() });
+    if (!rows.length)
+        return null;
+    const hours = Math.round((now.getTime() - Date.parse(st.lastDigestAt)) / 3600_000);
+    const title = `${w.name}: ${rows.length === 1 ? '1 update' : `${rows.length} updates`} from the last ${hours === 1 ? 'hour' : `${hours} hours`}`;
+    return sendNote(w, 'digest', `digest:${w.id}:${now.toISOString().slice(0, 13)}`, title, digestLines(rows).join('\n'), false, rows);
+}
+/** The morning note: what quiet hours held, what waits for the digest, and decisions still open. */
+export async function morningNote(w, now = deps.now()) {
+    const prefs = prefsFor(ME);
+    // Due when quiet hours end (08:00 with none), and for a few hours after, so a server that was busy or
+    // restarted at that minute still says good morning, and never while quiet hours still hold the phone.
+    const minute = now.getHours() * 60 + now.getMinutes();
+    const due = morningMinute(prefs);
+    if (minute < due || minute >= due + MORNING_WINDOW_MIN || inQuietHours(prefs, now))
+        return null;
+    const day = localDay(now);
+    const st = stateOf(w.id);
+    if (st.lastMorningDay === day)
+        return null;
+    update('update_state', w.id, { lastMorningDay: day });
+    const rows = pendingFor(w.id);
+    const held = rows.filter((u) => (u.held?.length ?? 0) > 0);
+    const waiting = liveCount(w.id);
+    // Since yesterday's morning, from the runs themselves (the same counts the daily digest uses).
+    const since = new Date(now.getTime() - 24 * 3600_000).toISOString();
+    const runs = digestRuns(list('runs', { watchId: w.id }).filter((r) => r.createdAt >= since));
+    if (!rows.length && !waiting)
+        return null;
+    // What needs the reader first: items to review, then what quiet hours held. "All green" never sits next to an open risk.
+    const open = list('approvals', { watchId: w.id, status: 'pending' }).filter((a) => !a.expiresAt || Date.parse(a.expiresAt) > now.getTime());
+    const items = open.slice(0, waiting).map(reviewItem);
+    while (items.length < waiting)
+        items.push('an item to review');
+    const note = morningText([...items, ...held.map((u) => plainLine(u.title, 80))], runs);
+    return sendNote(w, 'morning', `morning:${w.id}:${day}`, note.title, note.body, held.length > 0 || waiting > 0, rows);
+}
+export async function tick(now = deps.now()) {
+    try {
+        await retryRound(now);
+    }
+    catch (err) {
+        log.warn(`updates: retries failed: ${String(err)}`);
+    }
+    for (const w of list('watches', { paused: false })) {
+        if (w.archived)
+            continue;
+        try {
+            // The morning note first: it carries what waited, so the digest round finds nothing left.
+            await morningNote(w, now);
+            await digestRound(w, now);
+        }
+        catch (err) {
+            log.warn(`updates: digest for "${w.name}" failed: ${String(err)}`);
+        }
+    }
+}
+// ---------- pushes other flows ask for (KAT-281) ----------
+/**
+ * A phone push a flow sent through the push_notify rule (a digest ready, a rerun result, a QA review alert).
+ * The router makes no message of its own for most of them, so it sends them here: once per key, with retries,
+ * quiet hours, and the person's level. When the router already told the phone about the same run, or the
+ * run's own update is about to, this one is not sent, so the phone never gets two alerts for one run.
+ */
+export async function adoptPush(p) {
+    const draft = {
+        watchId: p.watchId,
+        kind: 'push',
+        key: `push:${p.key}`,
+        title: p.title,
+        body: p.body,
+        urgency: p.urgency,
+        ...(p.url ? { url: p.url } : {}),
+        ...(p.runId ? { runId: p.runId } : {}),
+    };
+    if (phoneAlreadyTold({ id: '', watchId: p.watchId, kind: 'push', ...(p.runId ? { runId: p.runId } : {}) }))
+        return null;
+    return handleDraft(draft, { level: 'now', reason: 'A flow asked for this push through the push_notify rule.' });
+}
+// ---------- start ----------
+/** Subscribes to live events and runs the digest clock. Returns stop. */
+export function startUpdates() {
+    // Sandbox mode (KAT-286): no digests and no morning notes. Direct senders stay, and they are refused at push and Slack.
+    if (sandboxOn()) {
+        sandboxSkips('the update router and its digests');
+        return () => undefined;
+    }
+    setUpdatesOwnDelivery(true);
+    setPushAdopter(async (p) => {
+        try {
+            await adoptPush(p);
+        }
+        catch (err) {
+            log.warn(`updates: push "${p.key}" failed: ${String(err)}`);
+        }
+    });
+    let chain = Promise.resolve();
+    const unsub = subscribe((e) => {
+        if (e.type !== 'run' && e.type !== 'approval' && e.type !== 'picked_up' && e.type !== 'dot')
+            return;
+        // One at a time, in event order, so a follow-up finds the message it follows.
+        chain = chain.then(() => handleEvent(e)).catch((err) => log.warn(`updates: ${String(err)}`));
+    });
+    const t = setInterval(() => void tick(), 60_000);
+    t.unref?.();
+    // Retries that were waiting when the server stopped go out soon after it starts, not a minute later.
+    const boot = setTimeout(() => void retryRound().catch((err) => log.warn(`updates: retries failed: ${String(err)}`)), 5_000);
+    boot.unref?.();
+    return () => {
+        unsub();
+        clearInterval(t);
+        clearTimeout(boot);
+        setUpdatesOwnDelivery(false);
+        setPushAdopter(null);
+    };
+}
+//# sourceMappingURL=index.js.map