tryharness 0.7.22 → 0.7.25

package/lib/watches.mjs

@@ -0,0 +1,362 @@
+import { spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+
+// Watches: something the person wants kept an eye on, read by a command on this computer on a schedule (a log, a status
+// command, a balance), and said out loud only when it is off. Each check reads one number from the command's output (a
+// field like "ageSeconds" or the count of lines matching a pattern like "critical") and compares it with a limit the
+// watch names, or, without one, with its own usual range once it has enough history. A falling number with a floor gets
+// a run-out estimate. Plain arithmetic, no model. An agent may suggest a watch; it runs only once the person starts it.
+// The October 13 keeper outage is why: its own health job logged "no recent successful keeper run" every 5 minutes for
+// 41 hours and nobody saw it.
+
+const HISTORY = 500;
+const USUAL_AFTER = 12;
+const MIN_EVERY_SECONDS = 5;
+const RUN_OUT_HOURS = 24;
+
+const median = (values) => {
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = sorted.length >> 1;
+  return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+};
+const escape = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+/**
+ * The number a check reads from output: how many lines hold a phrase (count), seconds since a named field's last
+ * timestamp (age: unix seconds, milliseconds or an ISO date), a named field's last value (field), or the last number in
+ * it. null when absent.
+ */
+export function readNumber(output, measure, now = Date.now()) {
+  const text = String(output ?? "");
+  if (measure.count)
+    return text
+      .split("\n")
+      .filter((line) =>
+        line.toLowerCase().includes(String(measure.count).toLowerCase())
+      ).length;
+  if (measure.age) {
+    const found = [
+      ...text.matchAll(
+        new RegExp(
+          `["']?${escape(measure.age)}["']?\\s*[:=]\\s*"?([0-9][0-9:.TZ+-]*)`,
+          "g"
+        )
+      ),
+    ].at(-1)?.[1];
+    if (!found) return null;
+    const at = /^\d+(?:\.\d+)?$/.test(found)
+      ? Number(found) * (Number(found) < 1e12 ? 1000 : 1)
+      : Date.parse(found);
+    return Number.isFinite(at)
+      ? Math.max(0, Math.round((now - at) / 1000))
+      : null;
+  }
+  if (measure.field) {
+    const found = [
+      ...text.matchAll(
+        new RegExp(
+          `["']?${escape(measure.field)}["']?\\s*[:=]\\s*"?(-?\\d+(?:\\.\\d+)?)`,
+          "g"
+        )
+      ),
+    ].at(-1);
+    return found ? Number(found[1]) : null;
+  }
+  const found = text.match(/-?\d+(?:\.\d+)?(?=\D*$)/);
+  return found ? Number(found[0]) : null;
+}
+
+/**
+ * Whether the newest sample is off, and why: past a limit the watch names, outside its usual range (median and spread of
+ * its history, once it has some), or, falling toward its floor, due to cross it within a day. null when it is fine.
+ */
+export function offPattern(samples, { above = null, below = null } = {}) {
+  const values = samples
+    .map((sample) => sample.value)
+    .filter((value) => value !== null && Number.isFinite(value));
+  const latest = samples.at(-1);
+  if (!latest || latest.value === null)
+    return latest?.error
+      ? { kind: "failed", text: `its check failed: ${latest.error}` }
+      : null;
+  if (above !== null && latest.value > above)
+    return { kind: "limit", text: `is ${latest.value}, above ${above}` };
+  if (below !== null && latest.value < below)
+    return { kind: "limit", text: `is ${latest.value}, below ${below}` };
+  if (below !== null && samples.length >= 4) {
+    // Falling toward the floor: a straight line through the recent samples, and when it reaches it.
+    const recent = samples.slice(-12).filter((sample) => sample.value !== null);
+    const t0 = recent[0].at;
+    const xs = recent.map((sample) => (sample.at - t0) / 3_600_000),
+      ys = recent.map((sample) => sample.value);
+    const mx = xs.reduce((a, b) => a + b, 0) / xs.length,
+      my = ys.reduce((a, b) => a + b, 0) / ys.length;
+    const slope =
+      xs.reduce((sum, x, i) => sum + (x - mx) * (ys[i] - my), 0) /
+      (xs.reduce((sum, x) => sum + (x - mx) ** 2, 0) || 1);
+    if (slope < 0) {
+      const hours = (latest.value - below) / -slope;
+      if (hours <= RUN_OUT_HOURS)
+        return {
+          kind: "run-out",
+          text: `is falling by ${Math.abs(slope).toPrecision(2)} an hour and reaches ${below} in about ${hours < 1 ? `${Math.max(1, Math.round(hours * 60))} minutes` : `${Math.round(hours)} hour${Math.round(hours) === 1 ? "" : "s"}`}`,
+        };
+    }
+  }
+  if (above === null && below === null && values.length > USUAL_AFTER) {
+    const history = values.slice(0, -1).slice(-100);
+    const middle = median(history);
+    // A number that takes two values (a sawtooth, a flag) has no median deviation; the mean deviation stands in.
+    const deviations = history.map((value) => Math.abs(value - middle));
+    const spread =
+      median(deviations) ||
+      deviations.reduce((a, b) => a + b, 0) / deviations.length ||
+      Math.max(1e-9, Math.abs(middle) * 0.05);
+    if (Math.abs(latest.value - middle) > 6 * spread)
+      return {
+        kind: "unusual",
+        text: `is ${latest.value}; it is usually about ${Number(middle.toPrecision(4))}`,
+      };
+  }
+  return null;
+}
+
+/** Runs a check's command in its folder; its output, or an error. A check never runs longer than its own interval. */
+function runCheck(watch, shell = process.env.SHELL || "/bin/sh") {
+  return new Promise((resolve) => {
+    const started = performance.now();
+    let out = "",
+      err = "",
+      done = false;
+    const child = spawn(shell, ["-lc", watch.command], {
+      cwd: watch.folder || undefined,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    const timer = setTimeout(() => {
+      child.kill("SIGKILL");
+      finish(null, `it took longer than its ${watch.everySeconds} s interval`);
+    }, watch.everySeconds * 1000);
+    const finish = (output, error) => {
+      if (done) return;
+      done = true;
+      clearTimeout(timer);
+      resolve({ output, error, ms: Math.round(performance.now() - started) });
+    };
+    child.stdout.on("data", (chunk) => {
+      out = (out + chunk).slice(-200_000);
+    });
+    child.stderr.on("data", (chunk) => {
+      err = (err + chunk).slice(-4_000);
+    });
+    child.on("error", (error) => finish(null, error.message));
+    child.on("close", (code) =>
+      finish(
+        code === 0 || out ? out : null,
+        code === 0 || out
+          ? null
+          : err.trim().split("\n").at(-1) || `it exited with code ${code}`
+      )
+    );
+  });
+}
+
+export function createWatches({
+  store,
+  notify = () => {},
+  log = () => {},
+  run = runCheck,
+}) {
+  const list = () => {
+    try {
+      return JSON.parse(store.meta("watches") || "[]");
+    } catch {
+      return [];
+    }
+  };
+  const save = (watches) => store.meta("watches", JSON.stringify(watches));
+  const samplesOf = (id) => {
+    try {
+      return JSON.parse(store.meta(`watch-samples:${id}`) || "[]");
+    } catch {
+      return [];
+    }
+  };
+  const timers = new Map();
+
+  const check = async (id) => {
+    const watch = list().find((entry) => entry.id === id);
+    if (!watch || watch.state !== "on") return null;
+    const result = await run(watch);
+    const value =
+      result.output === null ? null : readNumber(result.output, watch.measure);
+    const samples = [
+      ...samplesOf(id),
+      {
+        at: Date.now(),
+        value,
+        ms: result.ms,
+        ...(result.error
+          ? { error: result.error.slice(0, 300) }
+          : value === null
+            ? { error: "its output had no number to read" }
+            : {}),
+      },
+    ].slice(-HISTORY);
+    store.meta(`watch-samples:${id}`, JSON.stringify(samples));
+    const off = offPattern(samples, watch.measure);
+    const current = list().find((entry) => entry.id === id);
+    if (!current) return null;
+    // Said once when it goes off and once when it is back; not on every check while it stays off.
+    if (off && !current.off) {
+      const alert = {
+        at: new Date().toISOString(),
+        text: `${current.name} ${off.text}.`,
+        kind: off.kind,
+      };
+      save(
+        list().map((entry) =>
+          entry.id === id ? { ...entry, off: alert } : entry
+        )
+      );
+      store.meta(
+        "watch-alerts",
+        JSON.stringify(
+          [
+            ...JSON.parse(store.meta("watch-alerts") || "[]"),
+            { watch: id, ...alert },
+          ].slice(-200)
+        )
+      );
+      notify(alert.text, { watch: current, alert });
+      log(`watch ${current.name}: ${off.text}`);
+    } else if (!off && current.off) {
+      save(
+        list().map((entry) =>
+          entry.id === id ? { ...entry, off: null } : entry
+        )
+      );
+      notify(`${current.name} is back to normal.`, {
+        watch: current,
+        alert: null,
+      });
+    }
+    return { value, off, ms: result.ms };
+  };
+
+  const schedule = (watch) => {
+    clearTimeout(timers.get(watch.id));
+    if (watch.state !== "on") {
+      timers.delete(watch.id);
+      return;
+    }
+    const tick = () => {
+      void check(watch.id)
+        .catch((error) => log(`watch ${watch.name}: ${error.message}`))
+        .finally(() => {
+          if (timers.has(watch.id))
+            timers.set(
+              watch.id,
+              setTimeout(tick, watch.everySeconds * 1000).unref?.() ?? null
+            );
+        });
+    };
+    timers.set(watch.id, setTimeout(tick, 0));
+  };
+
+  return {
+    list: () =>
+      list().map((watch) => ({
+        ...watch,
+        samples: samplesOf(watch.id).slice(-20),
+      })),
+    alerts: (since = "") => {
+      try {
+        return JSON.parse(store.meta("watch-alerts") || "[]").filter(
+          (alert) => alert.at > since
+        );
+      } catch {
+        return [];
+      }
+    },
+    /** A new watch: the person's starts at once; an agent's waits for the person's start. */
+    add({
+      name,
+      command,
+      folder = null,
+      everySeconds = 300,
+      measure = {},
+      by = "user",
+    }) {
+      if (!String(name || "").trim() || !String(command || "").trim())
+        return {
+          ok: false,
+          error: "a watch needs a name and a command to read",
+        };
+      const every = Math.max(
+        MIN_EVERY_SECONDS,
+        Math.round(Number(everySeconds) || 300)
+      );
+      const clean = {
+        ...(measure.field
+          ? { field: String(measure.field).slice(0, 100) }
+          : {}),
+        ...(measure.count
+          ? { count: String(measure.count).slice(0, 100) }
+          : {}),
+        ...(measure.age ? { age: String(measure.age).slice(0, 100) } : {}),
+        above:
+          Number.isFinite(Number(measure.above)) &&
+          measure.above !== null &&
+          measure.above !== ""
+            ? Number(measure.above)
+            : null,
+        below:
+          Number.isFinite(Number(measure.below)) &&
+          measure.below !== null &&
+          measure.below !== ""
+            ? Number(measure.below)
+            : null,
+      };
+      const watch = {
+        id: randomUUID(),
+        name: String(name).trim().slice(0, 120),
+        command: String(command).slice(0, 2_000),
+        folder,
+        everySeconds: every,
+        measure: clean,
+        by,
+        state: by === "user" ? "on" : "suggested",
+        off: null,
+        createdAt: new Date().toISOString(),
+      };
+      save([...list(), watch]);
+      if (watch.state === "on") schedule(watch);
+      return { ok: true, watch };
+    },
+    start(id) {
+      const watch = list().find((entry) => entry.id === id);
+      if (!watch) return { ok: false, error: "no such watch" };
+      const on = { ...watch, state: "on" };
+      save(list().map((entry) => (entry.id === id ? on : entry)));
+      schedule(on);
+      return { ok: true, watch: on };
+    },
+    remove(id) {
+      if (!list().some((entry) => entry.id === id))
+        return { ok: false, error: "no such watch" };
+      clearTimeout(timers.get(id));
+      timers.delete(id);
+      save(list().filter((entry) => entry.id !== id));
+      store.meta(`watch-samples:${id}`, "[]");
+      return { ok: true };
+    },
+    check,
+    resume() {
+      for (const watch of list()) if (watch.state === "on") schedule(watch);
+    },
+    stop() {
+      for (const timer of timers.values()) clearTimeout(timer);
+      timers.clear();
+    },
+  };
+}