smart-home-engine 2.0.1 → 3.0.0-beta.0

package/src/index.js

@@ -71,7 +71,7 @@ const _pino = require('pino')(
     }),
 );
 // Lazy import — log-ws exports a no-op broadcastLog when the HTTP server is not started.
-const { broadcastLog, broadcast, setWelcomeProvider } = require('./web/log-ws');
+const { broadcastLog, broadcast, setWelcomeProvider, onClientMessage, onClientClose, sendWs } = require('./web/log-ws');
 const shedb = require('./web/shedb');
 // One level for every sink (roadmap B-12): a line below it is dropped before any work on it.
 const log = require('./lib/logger').createLogger({
@@ -237,6 +237,7 @@ if (typeof config.port !== 'undefined') {
             bindAddress: config.bindAddress,
             configPath: config.config,
             scriptDir: config.dir || null,
+            updateChannel: config.updateChannel,
         })
         .then((actualPort) => log.info('http server listening on :' + actualPort))
         .catch((err) => {
@@ -268,7 +269,18 @@ const { domain, vm, fs, path, suncalc } = modules;
 const scheduler = modules['node-schedule'];
 
 const StateStore = require('./lib/state-store');
-const { trackTimeout } = require('./lib/script-timers');
+const { trackTimeout, isValidDelay, describeDelay } = require('./lib/script-timers');
+const timing = require('./lib/timing');
+const timeLib = require('./lib/time');
+const publishChanged = require('./lib/publish-changed');
+const { Runtime } = require('./lib/runtime');
+const { callerLine } = require('./lib/call-site');
+const { createCommander } = require('./lib/command');
+const { statusTopicWarning, scanPublishTopics } = require('./lib/command-topics');
+// What every script has registered, with statistics — the Runtime page and the editor's panel
+const runtime = new Runtime();
+/** the script line a registration came from, or null */
+const lineOf = (file) => callerLine(file)?.line ?? null;
 const sandboxModules = [];
 const store = new StateStore();
 if (typeof config.port !== 'undefined') require('./web/broker-api').setStore(store);
@@ -282,9 +294,41 @@ const varSubscriptions = []; // store-based var:: subscriptions { key, handler,
 const scriptJobs = new Map(); // scriptFile → node-schedule Job[]
 const scriptPublishes = new Map(); // scriptFile → Set<topic> a script published to (roadmap I19; capped)
 const scriptTimers = new Map(); // scriptFile → Set<timer id>
+const scriptUnloadHooks = new Map(); // scriptFile → Set<fn> run first when the script is unloaded
 
 const _global = {};
 
+// One store listener serves every she.on('var::…') of every script. Before, each call
+// added a listener of its own, so Node warned about MaxListeners from the eleventh on; now a
+// registration is an entry in varSubscriptions, which unloadScript removes.
+store.on('change', (changedKey, val, obj, prevObj) => {
+    if (!changedKey.startsWith('var::')) return;
+    for (const sub of varSubscriptions.slice()) {
+        if (sub.key === changedKey) _dispatch(sub._script, () => sub.callback(val, obj, prevObj), sub._rt);
+    }
+});
+
+// she.mqtt.command waits for the device's status: one store listener serves every pending command
+const commandWatchers = new Map(); // topic → Set<cb(val)>
+store.on('change', (changedKey, val) => {
+    if (commandWatchers.size === 0 || !changedKey.startsWith('mqtt::')) return;
+    const set = commandWatchers.get(changedKey.slice(6));
+    if (!set) return;
+    for (const cb of [...set]) cb(val);
+});
+function watchTopic(topic, cb) {
+    let set = commandWatchers.get(topic);
+    if (!set) {
+        set = new Set();
+        commandWatchers.set(topic, set);
+    }
+    set.add(cb);
+    return () => {
+        set.delete(cb);
+        if (set.size === 0) commandWatchers.delete(topic);
+    };
+}
+
 // ── Event-loop heartbeat ─────────────────────────────────────────────────────
 let _blockingScript = null;
 let _blockingScriptTs = 0; // timestamp when the last _dispatch() call finished
@@ -297,18 +341,26 @@ let _blockingScriptTs = 0; // timestamp when the last _dispatch() call finished
  * as a last resort but cannot name the script; label-prefixed errors also show
  * up in the script's log panel in the Scripts tab.
  */
-function _dispatch(label, fn) {
+function _dispatch(label, fn, handler) {
     _blockingScript = label;
+    const t0 = handler ? performance.now() : 0;
     let result;
+    let failed = false;
     try {
         result = fn();
+    } catch (err) {
+        failed = true;
+        throw err;
     } finally {
         // Record end-time synchronously — no setImmediate, which would race with the
         // heartbeat timer when blocking occurs inside a poll-phase (I/O) callback.
         _blockingScriptTs = Date.now();
+        // the handler's statistics: fires, time, errors — since load and over the last five minutes
+        if (handler) handler.stats.record(performance.now() - t0, failed, _blockingScriptTs);
     }
     if (result && typeof result.then === 'function' && typeof result.catch === 'function') {
         result.catch((err) => {
+            if (handler) handler.stats.fail();
             const msg = err instanceof Error ? (err.stack ?? err.message) : String(err);
             log.error(makeLabel(label) + ' async callback error: ' + msg);
         });
@@ -337,6 +389,14 @@ const SUNCALC_EVENTS = new Set([
 const sunEvents = [];
 let sunTimes = [{}, /* today */ {}, /* tomorrow */ {}];
 
+// she.time: the zone, locale and holiday table every script's helpers use — validated once,
+// a bad value is logged once here and replaced by the default.
+const timeSettings = timeLib.settingsFromConfig(config, (msg) => log.warn(msg));
+log.info(
+    `she.time: zone ${timeSettings.timeZone}, locale ${timeSettings.locale}, holidays ${timeSettings.holidays || 'not configured'}` +
+        (timeSettings.holidaysExtra.length ? ` + ${timeSettings.holidaysExtra.length} extra` : ''),
+);
+
 function calculateSunTimes() {
     const now = new Date();
     const today = new Date(now.getFullYear(), now.getMonth(), now.getDate(), 12, 0, 0, 0, 0);
@@ -392,10 +452,10 @@ function sunScheduleEvent(obj, shift) {
                 // Event is less than 1s in the future or already in the past
                 // (options.random may have shifted us further to the past)
                 // call the callback immediately!
-                _dispatch(obj._script, obj.domain.bind(obj.callback));
+                _dispatch(obj._script, obj.domain.bind(obj.callback), obj._rt);
             } else {
                 // Schedule the event and track the job so it can be cancelled on script unload
-                obj._job = scheduler.scheduleJob(event, () => _dispatch(obj._script, obj.domain.bind(obj.callback)));
+                obj._job = scheduler.scheduleJob(event, () => _dispatch(obj._script, obj.domain.bind(obj.callback), obj._rt));
             }
         }
     }
@@ -517,11 +577,21 @@ new PerformanceObserver((list) => {
     }
 }).observe({ type: 'gc', buffered: false });
 
-// Register runtime stats provider for GET /she/status
-const statsSnapshot = () => {
+function countTopics() {
     let topics = 0;
     // eslint-disable-next-line no-unused-vars
     for (const _ of store.mqttEntries()) topics++;
+    return topics;
+}
+
+/**
+ * The daemon's gauges: rates since the previous sample (message rate, CPU, event-loop utilisation
+ * and lag), memory, counts. One sampler owns the interval counters — every reader (the status
+ * endpoint, the Runtime page, the hour's history) sees the same sample instead of resetting the
+ * counters under each other. Sampled every 10 s, every second while a Runtime page is open; the
+ * history keeps one sample per 10 s for an hour.
+ */
+function sampleGauges() {
     const now = Date.now();
     const elapsed = (now - _mqttMsgTs) / 1000;
     const mqttMsgPerSec = elapsed > 0 ? Math.round((_mqttMsgCount / elapsed) * 10) / 10 : 0;
@@ -532,11 +602,35 @@ const statsSnapshot = () => {
     const cpuPercent = elapsed > 0 ? Math.round(((cpuDelta.user + cpuDelta.system) / 1000 / (elapsed * 1000)) * 1000) / 10 : 0;
     const eluDelta = performance.eventLoopUtilization(_prevElu);
     _prevElu = performance.eventLoopUtilization();
-    const elMeanMs = Math.round(_elHisto.mean / 1e6);
-    const elMaxMs = Math.round(_elHisto.max / 1e6);
+    // the histogram is NaN before the first measurement of an interval — that is 0 lag, not null
+    const elMeanMs = Math.round(_elHisto.mean / 1e6) || 0;
+    const elMaxMs = Math.round(_elHisto.max / 1e6) || 0;
     _elHisto.reset();
     const mem = process.memoryUsage();
-    const memMb = Math.round(mem.rss / 1048576);
+    const sample = {
+        ts: now,
+        mqttMsgPerSec,
+        cpuPercent,
+        eluPercent: Math.round(eluDelta.utilization * 100),
+        elMeanMs,
+        elMaxMs,
+        memMb: Math.round(mem.rss / 1048576),
+        heapMb: Math.round(mem.heapUsed / 1048576),
+        topics: countTopics(),
+        handlers: runtime.handlers.size,
+        pendingTimers: runtime.pending.size,
+    };
+    runtime.latestGauges = sample;
+    runtime.gauges.push(sample);
+    return sample;
+}
+sampleGauges();
+
+// Register runtime stats provider for GET /she/status
+const statsSnapshot = () => {
+    const g = runtime.latestGauges || sampleGauges();
+    const { mqttMsgPerSec, cpuPercent, eluPercent, elMeanMs, elMaxMs, memMb, heapMb } = g;
+    const topics = countTopics();
     const core = shedb.getCore();
     const dbDocs = core ? Object.keys(core.docs).length : null;
     const dbViews = core ? Object.keys(core.queries).length : null;
@@ -571,16 +665,79 @@ const statsSnapshot = () => {
         dbViews,
         handlers: subscriptions.length + varSubscriptions.length + mqttEventCallbacks.length,
         memMb,
+        heapMb,
         cpuPercent,
-        eluPercent: Math.round(eluDelta.utilization * 100),
+        eluPercent,
         elMeanMs,
         elMaxMs,
+        pendingTimers: runtime.pending.size,
+        commandsFailed: runtime.commandsFailed,
         safeMode: SAFE_MODE,
         safeModeReason: SAFE_MODE_REASON,
     };
 };
 require('./web/server').setStatsProvider(statsSnapshot);
 
+// The Runtime page and the editor's Runtime panel: a snapshot about once a second over the
+// WebSocket, only to the clients that asked for it (optionally one script's part); the gauges are
+// sampled every second while someone watches, every 10 s otherwise.
+const _runtimeSubs = new Map(); // ws → script file | null (everything)
+const scriptLabel = (f) => makeLabel(f).slice(0, -1);
+const scriptOrigin = (f) => scriptOrigins.get(f) || 'user';
+const fileOfLabel = (label) => Object.keys(scripts).find((f) => scriptLabel(f) === label);
+function runtimeSnapshot(file) {
+    return runtime.snapshot({ script: file, label: scriptLabel, origin: scriptOrigin });
+}
+function pushRuntime() {
+    if (_runtimeSubs.size === 0) return;
+    let full = null;
+    for (const [ws, file] of _runtimeSubs) {
+        let snap;
+        if (file === null) snap = full || (full = runtimeSnapshot());
+        else snap = runtimeSnapshot(file);
+        sendWs(ws, { type: 'runtime', ...snap });
+    }
+}
+let _sampleTimer = null;
+function scheduleSample() {
+    if (_sampleTimer) clearTimeout(_sampleTimer);
+    _sampleTimer = setTimeout(
+        () => {
+            sampleGauges();
+            pushRuntime();
+            scheduleSample();
+        },
+        _runtimeSubs.size > 0 ? 1000 : 10_000,
+    );
+    _sampleTimer.unref();
+}
+scheduleSample();
+onClientMessage('runtime:subscribe', (ws, msg) => {
+    let file = null;
+    if (typeof msg.script === 'string' && msg.script) {
+        file = fileOfLabel(msg.script);
+        // not loaded (a library, a disabled or broken script): subscribe anyway and show it empty — the
+        // panel re-subscribes when the script's running state changes
+        if (!file) file = '\0unknown:' + msg.script;
+    }
+    const first = _runtimeSubs.size === 0;
+    _runtimeSubs.set(ws, file);
+    sendWs(ws, { type: 'runtime', ...runtimeSnapshot(file) });
+    if (first) scheduleSample(); // from the 10 s to the 1 s cadence at once
+});
+onClientMessage('runtime:unsubscribe', (ws) => _runtimeSubs.delete(ws));
+onClientClose((ws) => _runtimeSubs.delete(ws));
+require('./web/runtime-api').init({
+    runtime,
+    snapshot: (label) => {
+        if (!label) return runtimeSnapshot();
+        const file = fileOfLabel(label);
+        return runtimeSnapshot(file || '\0unknown:' + label);
+    },
+    history: () => ({ intervalMs: runtime.gauges.intervalMs, samples: runtime.gauges.list() }),
+    log: (file, msg) => log.info(makeLabel(file), msg),
+});
+
 // Health probe for GET /she/health (roadmap A1) — cheap, no counting, no allocation.
 const healthSnapshot = () => ({
     started: _started,
@@ -609,7 +766,14 @@ const introspect = {
             if (!mqtt || !connected) return reject(new Error('MQTT is not connected'));
             mqtt.publish(String(topic), typeof payload === 'object' ? JSON.stringify(payload) : String(payload), opts || {}, (err) => (err ? reject(err) : resolve()));
         }),
-    config: () => ({ name: config.name, variablePrefix: config.variablePrefix || 'var', version: pkg.version }),
+    config: () => ({
+        name: config.name,
+        variablePrefix: config.variablePrefix || 'var',
+        version: pkg.version,
+        timezone: timeSettings.timeZone,
+        locale: timeSettings.locale,
+        holidays: timeSettings.holidays,
+    }),
     scripts: () =>
         Object.keys(scripts).map((file) => ({
             file,
@@ -620,7 +784,13 @@ const introspect = {
             publishes: [...(scriptPublishes.get(file) || [])],
             jobs: (scriptJobs.get(file) || []).filter(Boolean).map((job) => ({ next: nextOf(job) })),
             sunEvents: sunEvents.filter((e) => e._script === file).map((e) => ({ pattern: e.pattern, next: nextOf(e._job) })),
-            timers: [...(scriptTimers.get(file) || [])].map((t) => ({ due: t.due || null, every: t.every || null })),
+            timers: [...(scriptTimers.get(file) || [])].map((t) => ({
+                due: t.due || null,
+                start: t.start || null,
+                every: t.every || null,
+                kind: t.kind || null,
+                name: t.name || null,
+            })),
         })),
     stats: statsSnapshot,
     health: healthSnapshot,
@@ -935,17 +1105,50 @@ function stateChange(topic, state, oldState, msg) {
 
             delay = Math.floor(delay);
 
-            setTimeout(() => {
-                /**
-                 * @callback subscribeCallback
-                 * @param {string} topic - the topic that triggered this callback
-                 * @param {mixed} val - the val property of the new state
-                 * @param {object} obj - new state - the whole state object (e.g. {"val": true, "ts": 12346345, "lc": 12346345} )
-                 * @param {object} objPrev - previous state - the whole state object
-                 * @param {object} msg - the mqtt message as received from MQTT.js
-                 */
-                _dispatch(subs._script, () => subs.callback(topic, state.val, state, oldState, msg));
-            }, delay);
+            /**
+             * @callback subscribeCallback
+             * @param {string} topic - the topic that triggered this callback
+             * @param {mixed} val - the val property of the new state
+             * @param {object} obj - new state - the whole state object (e.g. {"val": true, "ts": 12346345, "lc": 12346345} )
+             * @param {object} objPrev - previous state - the whole state object
+             * @param {object} msg - the mqtt message as received from MQTT.js
+             */
+            const run = () => subs.deliver(topic, state.val, state, oldState, msg);
+            const timers = scriptTimers.get(subs._script) || new Set();
+            // The delay timer is tracked like a script's own setTimeout, so a reload of
+            // the script between the message and the shifted callback cancels the callback instead
+            // of running the unloaded script's code.
+            if (delay === 0) {
+                trackTimeout(timers, run, 0);
+            } else {
+                // a shifted delivery is a pending timer of the script: listed under the
+                // subscription's `shift` group, cancellable from the editor
+                const group = runtime.timerGroup(subs._script, { kind: 'shift', name: subs.topic, line: subs._rt?.line ?? null });
+                let pending = null;
+                const id = trackTimeout(
+                    timers,
+                    () => {
+                        runtime.removePending(pending);
+                        run();
+                    },
+                    delay,
+                    { kind: 'shift', name: subs.topic },
+                );
+                const clear = () => {
+                    timers.delete(id);
+                    clearTimeout(id);
+                    runtime.removePending(pending);
+                };
+                pending = runtime.addPending(group, {
+                    due: id.due,
+                    start: id.start,
+                    fire: () => {
+                        clear();
+                        run();
+                    },
+                    cancel: clear,
+                });
+            }
         }
     });
 }
@@ -995,7 +1198,7 @@ function createScript(source, name) {
     }
 }
 
-function runScript(script, name, _origin) {
+function runScript(script, name, _origin, source) {
     const scriptDir = path.dirname(path.resolve(name));
     const logLabel = makeLabel(name);
 
@@ -1004,6 +1207,14 @@ function runScript(script, name, _origin) {
     if (!scriptTimers.has(name)) scriptTimers.set(name, new Set());
     const _myJobs = scriptJobs.get(name);
     const _myTimers = scriptTimers.get(name);
+    const _myHooks = new Set();
+    scriptUnloadHooks.set(name, _myHooks);
+    const _myLastPayload = new Map(); // topic → last payload, for pub({ changed: true })
+    let _unloaded = false;
+    let _subCount = 0;
+    _myHooks.add(() => {
+        _unloaded = true;
+    });
 
     log.debug(logLabel, 'creating domain');
     const scriptDomain = domain.create();
@@ -1014,6 +1225,100 @@ function runScript(script, name, _origin) {
         return a !== null && typeof a === 'object' ? JSON.stringify(a) : a;
     }
 
+    /**
+     * The script's one-shot timer: the delay is validated and a bad one is logged with the script's
+     * name, the callback runs in the script's domain, the id is tracked for the unload
+     * and carries kind and name for the timer list. Every timer API of the sandbox goes through here.
+     */
+    function _timeout(fn, delay, meta = {}, api = 'setTimeout') {
+        if (_unloaded) {
+            log.debug(logLabel, `${api} after unload ignored`);
+            return null;
+        }
+        // The runtime model groups timers by kind, name and the line that created them: a plain
+        // setTimeout by its call site, a timing helper by the line the helper was created on (the
+        // helper passes it in its meta — the stack at re-arm time points into a callback).
+        const kind = meta.kind || 'timer';
+        const line = meta.line !== undefined ? meta.line : kind === 'timer' ? lineOf(name) : null;
+        const group = runtime.timerGroup(name, { kind, name: meta.name, line });
+        const run = scriptDomain.bind(() => _dispatch(name, fn, group));
+        let pending = null;
+        const id = trackTimeout(
+            _myTimers,
+            () => {
+                runtime.removePending(pending);
+                run();
+            },
+            delay,
+            {
+                warn: (msg) => log.warn(logLabel, msg),
+                api,
+                kind: meta.kind,
+                name: meta.name,
+            },
+        );
+        pending = runtime.addPending(group, {
+            due: id.due,
+            start: id.start,
+            fire:
+                meta.actions && meta.actions.run === false
+                    ? null
+                    : () => {
+                          _clearTimeout(id);
+                          run();
+                      },
+            cancel: () => {
+                _clearTimeout(id);
+                if (typeof meta.onCancel === 'function') meta.onCancel();
+            },
+            extra: meta.extra,
+        });
+        id.rt = pending;
+        return id;
+    }
+    function _clearTimeout(id) {
+        _myTimers.delete(id);
+        clearTimeout(id);
+        if (id && id.rt) runtime.removePending(id.rt);
+    }
+    const _timing = timing.createTimingHelpers({
+        setTimeout: (fn, ms, meta) => _timeout(fn, ms, meta),
+        clearTimeout: _clearTimeout,
+        onUnload: (fn) => {
+            _myHooks.add(fn);
+            return () => _myHooks.delete(fn);
+        },
+        line: () => lineOf(name),
+    });
+    const _lintWarned = new Set(); // topics already warned about (load-time scan and runtime)
+    function lintPublish(topic, api, line) {
+        if (_lintWarned.has(topic)) return;
+        const why = statusTopicWarning(topic, (t) => store.has('mqtt::' + t));
+        if (!why) return;
+        _lintWarned.add(topic);
+        log.warn(logLabel, `${api}${line ? ` (line ${line})` : ''}: ${why}`);
+    }
+    const _command = createCommander({
+        publish: (topic, payload) => {
+            recordPublish(name, topic);
+            if (mqtt && connected) mqtt.publish(topic, payload, {});
+        },
+        get: (topic) => store.get('mqtt::' + topic),
+        has: (topic) => store.has('mqtt::' + topic),
+        watch: watchTopic,
+        setTimeout: (fn, ms, meta) => _timeout(fn, ms, meta, 'she.mqtt.command'),
+        clearTimeout: _clearTimeout,
+        onUnload: (fn) => {
+            _myHooks.add(fn);
+            return () => _myHooks.delete(fn);
+        },
+        warn: (msg) => log.warn(logLabel, msg),
+        onFail: () => {
+            runtime.commandsFailed++;
+        },
+        line: () => lineOf(name),
+    });
+
     const she = {
         global: _global,
 
@@ -1087,36 +1392,59 @@ function runScript(script, name, _origin) {
             } else if (rest.length > 2) {
                 throw new Error('wrong number of arguments');
             }
+            options = options || {};
 
-            if (typeof topic === 'string') {
-                if (typeof options.condition === 'string') {
-                    if (options.condition.indexOf('\n') !== -1) {
-                        throw new Error('options.condition string must be one-line javascript');
-                    }
-                    /* eslint-disable no-new-func */
-                    options.condition = new Function('topic', 'val', 'obj', 'objPrev', 'msg', 'return ' + options.condition + ';');
-                }
+            let topics;
+            if (typeof topic === 'string') topics = [topic];
+            else if (typeof topic === 'object' && topic.length > 0) topics = Array.prototype.slice.call(topic);
+            else return;
 
-                if (typeof options.condition === 'function') {
-                    options.condition = scriptDomain.bind(options.condition);
+            if (typeof options.condition === 'string') {
+                if (options.condition.indexOf('\n') !== -1) {
+                    throw new Error('options.condition string must be one-line javascript');
                 }
+                /* eslint-disable no-new-func */
+                options.condition = new Function('topic', 'val', 'obj', 'objPrev', 'msg', 'return ' + options.condition + ';');
+            }
+
+            if (typeof options.condition === 'function') {
+                options.condition = scriptDomain.bind(options.condition);
+            }
 
-                subscriptions.push({ topic, options, callback: typeof callback === 'function' && scriptDomain.bind(callback), _script: name });
+            const boundCallback = typeof callback === 'function' && scriptDomain.bind(callback);
+            const label = topics.length > 1 ? `${topics[0]} +${topics.length - 1}` : topics[0];
+            // one handler in the runtime model per call — the topics of an array share it, like
+            // they share the limiter below
+            const optNames = ['change', 'retain', 'shift', 'random', 'condition', 'debounce', 'throttle', 'maxWait', 'perTopic'].filter(
+                (k) => options[k] !== undefined && options[k] !== false,
+            );
+            const detail = [topics.length > 1 ? topics.join(', ') : null, optNames.length ? optNames.join(', ') : null].filter(Boolean).join(' · ') || null;
+            const handler = runtime.handler(name, { kind: 'sub', name: label, detail, line: lineOf(name) });
+            // what runs the callback for one event; with opts.debounce or opts.throttle the events of
+            // this call are bundled — across the topics of an array, per topic with opts.perTopic
+            let deliver = (t, val, obj, objPrev, msg) => _dispatch(name, () => boundCallback(t, val, obj, objPrev, msg), handler);
+            const limited = timing.subscriptionLimiter(options, deliver, _timing, label);
+            if (limited) deliver = limited;
+
+            for (const tp of topics) {
+                subscriptions.push({ topic: tp, options, callback: boundCallback, deliver, _script: name, _rt: handler });
+                _subCount++;
+                if (_subCount === 1000) {
+                    log.warn(
+                        logLabel,
+                        'has registered 1000 subscriptions — she.mqtt.sub is meant to be called when the script loads, not inside a callback: every call adds a subscription that stays until the script is unloaded',
+                    );
+                }
 
-                if (options.retain && store.has('mqtt::' + topic) && typeof callback === 'function') {
-                    _dispatch(name, () => callback(topic, store.get('mqtt::' + topic), store.getObject('mqtt::' + topic)));
-                } else if (options.retain && (/\/\+\//.test(topic) || /\+$/.test(topic) || /\+/.test(topic) || topic.endsWith('#')) && typeof callback === 'function') {
-                    for (const [t, obj] of store.mqttEntries()) {
-                        if (mqttWildcards(t, topic)) {
-                            _dispatch(name, () => callback(t, obj.val, obj));
+                if (options.retain && boundCallback) {
+                    if (store.has('mqtt::' + tp)) {
+                        deliver(tp, store.get('mqtt::' + tp), store.getObject('mqtt::' + tp));
+                    } else if (tp.includes('+') || tp.endsWith('#')) {
+                        for (const [t, obj] of store.mqttEntries()) {
+                            if (mqttWildcards(t, tp)) deliver(t, obj.val, obj);
                         }
                     }
                 }
-            } else if (typeof topic === 'object' && topic.length > 0) {
-                topic = Array.prototype.slice.call(topic);
-                topic.forEach((tp) => {
-                    she.mqttsub(tp, options, callback);
-                });
             }
         },
 
@@ -1166,6 +1494,7 @@ function runScript(script, name, _origin) {
                 if (typeof options.shift !== 'undefined' && (options.shift < -86400 || options.shift > 86400)) {
                     throw new Error('options.shift out of range');
                 }
+                const sunDetail = [options.shift ? `shift ${options.shift}s` : null, options.random ? `random ${options.random}s` : null].filter(Boolean).join(', ') || null;
                 const obj = {
                     pattern,
                     options,
@@ -1173,23 +1502,61 @@ function runScript(script, name, _origin) {
                     context: she,
                     domain: scriptDomain,
                     _script: name,
+                    _rt: null,
                 };
+                obj._rt = runtime.handler(name, {
+                    kind: 'sun',
+                    name: pattern,
+                    detail: sunDetail,
+                    line: lineOf(name),
+                    next: () => nextOf(obj._job),
+                    run: () => _dispatch(name, scriptDomain.bind(callback), obj._rt),
+                });
                 sunEvents.push(obj);
                 sunScheduleEvent(obj);
                 return;
             }
 
-            if (options.random) {
-                _myJobs.push(
-                    scheduler.scheduleJob(pattern, () => {
-                        // Track the random-delay timer so it is cancelled on unload
-                        // if the job fires in the same tick as the script is reloaded.
-                        trackTimeout(_myTimers, () => _dispatch(name, scriptDomain.bind(callback)), (parseFloat(options.random) || 0) * 1000 * Math.random());
-                    }),
-                );
-            } else {
-                _myJobs.push(scheduler.scheduleJob(pattern, () => _dispatch(name, scriptDomain.bind(callback))));
+            // a Date from the sandbox is of the context's realm, so no instanceof
+            const shown = pattern && typeof pattern.toISOString === 'function' ? pattern.toISOString() : typeof pattern === 'string' ? pattern : JSON.stringify(pattern);
+            let handler = null;
+            const runNow = () => _dispatch(name, scriptDomain.bind(callback), handler);
+            const run = options.random
+                ? () => {
+                      // Track the random-delay timer so it is cancelled on unload
+                      // if the job fires in the same tick as the script is reloaded.
+                      _timeout(
+                          () => runNow(),
+                          (parseFloat(options.random) || 0) * 1000 * Math.random(),
+                          { kind: 'schedule random', name: shown, line: handler?.line ?? null },
+                          'she.schedule',
+                      );
+                  }
+                : runNow;
+            let job = null;
+            job = scheduler.scheduleJob(pattern, () => {
+                // A one-shot job (a Date) is done after this run: forget it, so that a script
+                // scheduling a Date per event does not keep every fired job.
+                if (job && job.nextInvocation() === null) {
+                    const i = _myJobs.indexOf(job);
+                    if (i !== -1) _myJobs.splice(i, 1);
+                    runtime.removeHandler(handler);
+                }
+                run();
+            });
+            if (!job) {
+                log.warn(logLabel, `she.schedule: nothing scheduled for ${shown} — not a valid pattern, or a date in the past`);
+                return;
             }
+            _myJobs.push(job);
+            handler = runtime.handler(name, {
+                kind: 'schedule',
+                name: shown,
+                detail: options.random ? `random ${options.random}s` : null,
+                line: lineOf(name),
+                next: () => nextOf(job),
+                run: runNow,
+            });
         },
 
         /**
@@ -1200,6 +1567,8 @@ function runScript(script, name, _origin) {
          * @param {Object} [options] - the options to publish with
          * @param {number} [options.qos=0] - QoS Level
          * @param {boolean} [options.retain=false] - retain flag
+         * @param {boolean} [options.changed=false] - publish only when the payload differs from the last one this
+         *   script published to the topic (before the first one: from the topic's current value in the state store)
          */
         mqttpub: function Sandbox_mqttpub(topic, payload, options) {
             if (typeof topic === 'object' && topic.length > 0) {
@@ -1212,16 +1581,36 @@ function runScript(script, name, _origin) {
 
             topic = topic.replace(/^([^/]+)\/\/(.+)$/, '$1/set/$2');
             recordPublish(name, topic);
+            lintPublish(topic, 'she.mqtt.pub');
 
             if (typeof payload === 'object') {
                 payload = JSON.stringify(payload);
             } else {
                 payload = String(payload);
             }
+            let changed = false;
+            if (options && typeof options === 'object' && 'changed' in options) {
+                ({ changed, ...options } = options);
+            }
+            if (changed && publishChanged.isRepeat(_myLastPayload, topic, payload, (t) => store.getObject('mqtt::' + t))) return;
             if (!mqtt || !connected) return; // silently drop when MQTT is not connected
+            publishChanged.remember(_myLastPayload, topic, payload);
             mqtt.publish(topic, payload, options);
         },
 
+        /**
+         * Publish a command and wait for the device's status to confirm it. she.mqtt.command.
+         * @method mqttcommand
+         * @param {string} topic
+         * @param {*} payload
+         * @param {Object} [options] confirm, path, expect, tolerance, timeout (s), retries, quiet
+         * @returns {Promise<{val, ms, tries, already?}>}
+         */
+        mqttcommand: function Sandbox_mqttcommand(topic, payload, options) {
+            if (typeof topic === 'string') lintPublish(topic.replace(/^([^/]+)\/\/(.+)$/, '$1/set/$2'), 'she.mqtt.command');
+            return _command(topic, payload, options);
+        },
+
         /**
          * Set a value on one or more topics
          * @method setValue
@@ -1308,14 +1697,12 @@ function runScript(script, name, _origin) {
             } else if (key.startsWith('var::')) {
                 const varStoreKey = 'var::' + key.slice(5);
                 const boundCb = scriptDomain.bind(callback);
-                const varHandler = (changedKey, val, obj, prevObj) => {
-                    if (changedKey === varStoreKey) _dispatch(name, () => boundCb(val, obj, prevObj));
-                };
-                store.on('change', varHandler);
-                varSubscriptions.push({ key: varStoreKey, handler: varHandler, _script: name });
+                const handler = runtime.handler(name, { kind: 'var', name: key, line: lineOf(name) });
+                // served by the one store listener at module level
+                varSubscriptions.push({ key: varStoreKey, callback: boundCb, _script: name, _rt: handler });
                 // Fire immediately if var already has a value (retain semantics)
                 const currentVarObj = store.getObject(varStoreKey);
-                if (currentVarObj !== undefined) boundCb(currentVarObj.val, currentVarObj, undefined);
+                if (currentVarObj !== undefined) _dispatch(name, () => boundCb(currentVarObj.val, currentVarObj, undefined), handler);
             } else if (key.startsWith('matter::')) {
                 const parts = key.slice(8).split('/');
                 if (parts.length !== 4) throw new TypeError('she.on: invalid matter key (expected matter::nodeId/ep/Cluster/attr)');
@@ -1332,31 +1719,60 @@ function runScript(script, name, _origin) {
     she.log = she.info;
 
     const Sandbox = {
-        setTimeout: (fn, delay, ...args) => {
+        // setTimeout(fn) without a delay is Node's "next tick"; a delay that is given but is not a
+        // number is a mistake and logged with the script's name
+        setTimeout: (fn, ...rest) => {
+            const [delay, ...args] = rest;
             const wrapped = args.length ? () => fn(...args) : fn;
-            return trackTimeout(_myTimers, () => _dispatch(name, wrapped), delay);
+            return _timeout(wrapped, rest.length ? delay : 0);
         },
-        setInterval: (fn, delay, ...args) => {
+        setInterval: (fn, ...rest) => {
+            let [delay, ...args] = rest;
+            if (!isValidDelay(delay)) {
+                log.warn(logLabel, `setInterval: delay is not a number of milliseconds (got ${describeDelay(delay)}), using 0`);
+                delay = 0;
+            }
             const wrapped = args.length ? () => fn(...args) : fn;
-            const id = setInterval(() => _dispatch(name, wrapped), delay);
+            let handler = null;
+            let pending = null;
+            const tick = scriptDomain.bind(() => _dispatch(name, wrapped, handler));
+            const id = setInterval(() => {
+                // the pending entry is the next run: the progress bar starts over with every tick
+                if (pending) {
+                    pending.start = Date.now();
+                    pending.due = pending.start + delay;
+                }
+                tick();
+            }, delay);
             id.every = delay;
-            id.due = Date.now() + delay;
+            id.start = Date.now();
+            id.due = id.start + delay;
             _myTimers.add(id);
+            const clear = () => {
+                _myTimers.delete(id);
+                clearInterval(id);
+                runtime.removeHandler(handler);
+            };
+            handler = runtime.handler(name, { kind: 'interval', name: `every ${delay} ms`, line: lineOf(name), run: tick, cancel: clear });
+            pending = runtime.addPending(handler, { due: id.due, start: id.start, every: delay });
+            id.rt = handler;
             return id;
         },
-        clearTimeout: (id) => {
-            _myTimers.delete(id);
-            clearTimeout(id);
-        },
+        clearTimeout: _clearTimeout,
         clearInterval: (id) => {
             _myTimers.delete(id);
             clearInterval(id);
+            if (id && id.rt) runtime.removeHandler(id.rt);
         },
 
         Buffer,
 
         require(md) {
-            if (modules[md]) {
+            // A relative path is cached by Node under its resolved file; caching it here under the bare
+            // specifier made two scripts requiring './util' from different directories share the first
+            // one's module. The cache here is for the daemon's own modules and packages.
+            const relative = /^\.\.?\//.test(md);
+            if (!relative && modules[md]) {
                 return modules[md];
             }
             // Failures throw (Cannot find module, syntax errors in the required
@@ -1384,7 +1800,7 @@ function runScript(script, name, _origin) {
                     }
                 }
             }
-            modules[md] = result;
+            if (!relative) modules[md] = result;
             return result;
         },
 
@@ -1401,6 +1817,9 @@ function runScript(script, name, _origin) {
     she.config = Object.freeze({
         latitude: config.latitude,
         longitude: config.longitude,
+        timezone: timeSettings.timeZone,
+        locale: timeSettings.locale,
+        holidays: timeSettings.holidays,
     });
     // she.secrets (roadmap A5): read-only access to the secrets store; unknown paths warn once per script
     const _secretWarned = new Set();
@@ -1415,20 +1834,33 @@ function runScript(script, name, _origin) {
         },
         has: (p) => secrets.has(p),
     });
-    // she.setTimeout / she.clearTimeout — tracked versions for use by stdlib and
-    // sandbox modules that don't have direct access to the Sandbox context.
-    she.setTimeout = (fn, delay, ...args) => {
-        const wrapped = args.length ? () => fn(...args) : fn;
-        return trackTimeout(_myTimers, () => _dispatch(name, wrapped), delay);
-    };
-    she.clearTimeout = (id) => {
-        _myTimers.delete(id);
-        clearTimeout(id);
-    };
+    // she.setTimeout / she.clearTimeout — the sandbox's tracked timers under the she object: the
+    // documented aliases of the globals, and what stdlib and the sandbox modules use.
+    she.setTimeout = Sandbox.setTimeout;
+    she.clearTimeout = Sandbox.clearTimeout;
+    // Timing helpers: their timers are the script's, so a reload clears them and
+    // the timer list shows them with kind and name.
+    she.debounce = _timing.debounce;
+    she.throttle = _timing.throttle;
+    she.timeout = _timing.timeout;
+    she.retry = _timing.retry;
+    // she.time: windows, wall-clock instants, holidays and formatting in the configured zone;
+    // sun events come from suncalc with the coordinates she.schedule uses.
+    she.time = timeLib.createTime({
+        ...timeSettings,
+        sunTimes: (noonMs) => suncalc.getTimes(new Date(noonMs), config.latitude, config.longitude),
+        warn: (msg) => log.warn(logLabel, msg),
+    });
     sandboxModules.forEach((md) => {
         md(she, { scriptDomain, scriptName, scriptFile: name });
     });
 
+    // Lint before the script runs: a literal topic that an adapter writes as status is a command
+    // that will never arrive (<inst>/status/… instead of <inst>/set/…)
+    if (source !== undefined) {
+        for (const hit of scanPublishTopics(source)) lintPublish(hit.topic, hit.api, hit.line);
+    }
+
     log.debug(logLabel, 'contextifying sandbox');
     const context = vm.createContext(Sandbox);
 
@@ -1506,7 +1938,7 @@ function loadScript(file, origin) {
             }
             if (scripts[file]) {
                 scriptOrigins.set(file, origin);
-                runScript(scripts[file], file, origin);
+                runScript(scripts[file], file, origin, src.toString());
                 broadcast({ type: 'script:running', path: makeLabel(file).slice(0, -1), running: true });
             }
         }
@@ -1523,6 +1955,20 @@ function unloadScript(file) {
     let removedCallbacks = 0;
     let removedTimers = 0;
 
+    // The script's unload hooks run first: the sandbox marks itself unloaded, a debounce
+    // with flushOnUnload runs its pending call, a retry stops. Then the timers below are cleared.
+    const hooks = scriptUnloadHooks.get(file);
+    if (hooks) {
+        scriptUnloadHooks.delete(file);
+        for (const hook of hooks) {
+            try {
+                _dispatch(file, hook);
+            } catch (err) {
+                log.error(unloadLabel, 'error while unloading: ' + (err && err.stack ? err.stack : err));
+            }
+        }
+    }
+
     // Remove MQTT subscriptions belonging to this script
     for (let i = subscriptions.length - 1; i >= 0; i--) {
         if (subscriptions[i]._script === file) {
@@ -1568,11 +2014,11 @@ function unloadScript(file) {
         scriptTimers.delete(file);
     }
     scriptPublishes.delete(file);
+    runtime.removeScript(file);
 
     // Remove store-based var:: subscriptions belonging to this script
     for (let i = varSubscriptions.length - 1; i >= 0; i--) {
         if (varSubscriptions[i]._script === file) {
-            store.removeListener('change', varSubscriptions[i].handler);
             varSubscriptions.splice(i, 1);
             removedCallbacks++;
         }