smart-home-engine 1.21.0 → 1.34.3

package/src/index.js

@@ -37,6 +37,12 @@ if (process.argv.includes('--install')) {
 // Ensure the data directory exists before anything else runs
 require('./lib/storage').ensureRoot();
 
+// Secrets CLI (roadmap A5): she --secret-set <group>/<field> (value on stdin) | --secret-delete <group>[/<field>] | --secret-list
+{
+    const idx = process.argv.findIndex((a) => a === '--secret-set' || a === '--secret-delete' || a === '--secret-list');
+    if (idx !== -1) process.exit(require('./lib/secrets').cli(process.argv.slice(idx)));
+}
+
 // ---------------------------------------------------------------------------
 // Persistent JSON-Lines log file — written alongside the pino-pretty stream.
 // On each daemon start: rotate she.jsonl → she.jsonl.1, then open fresh.
@@ -44,6 +50,7 @@ require('./lib/storage').ensureRoot();
 const _fs = require('fs');
 const _path = require('path');
 const { LOGS_DIR } = require('./lib/storage');
+const secrets = require('./lib/secrets');
 const _logFileCurrent = _path.join(LOGS_DIR, 'she.jsonl');
 const _logFilePrev = _path.join(LOGS_DIR, 'she.jsonl.1');
 try {
@@ -78,29 +85,40 @@ const { broadcastLog, broadcast, setWelcomeProvider } = require('./web/log-ws');
 const shedb = require('./web/shedb');
 const log = {
     debug: (...args) => {
-        _pino.debug(args.join(' '));
-        broadcastLog({ level: 'debug', msg: args.join(' '), ts: Date.now() });
-        _writeLogLine('debug', args.join(' '));
+        const msg = secrets.redact(args.join(' '));
+        _pino.debug(msg);
+        broadcastLog({ level: 'debug', msg, ts: Date.now() });
+        _writeLogLine('debug', msg);
     },
     info: (...args) => {
-        _pino.info(args.join(' '));
-        broadcastLog({ level: 'info', msg: args.join(' '), ts: Date.now() });
-        _writeLogLine('info', args.join(' '));
+        const msg = secrets.redact(args.join(' '));
+        _pino.info(msg);
+        broadcastLog({ level: 'info', msg, ts: Date.now() });
+        _writeLogLine('info', msg);
     },
     warn: (...args) => {
-        _pino.warn(args.join(' '));
-        broadcastLog({ level: 'warn', msg: args.join(' '), ts: Date.now() });
-        _writeLogLine('warn', args.join(' '));
+        const msg = secrets.redact(args.join(' '));
+        _pino.warn(msg);
+        broadcastLog({ level: 'warn', msg, ts: Date.now() });
+        _writeLogLine('warn', msg);
     },
     error: (...args) => {
-        _pino.error(args.join(' '));
-        broadcastLog({ level: 'error', msg: args.join(' '), ts: Date.now() });
-        _writeLogLine('error', args.join(' '));
+        const msg = secrets.redact(args.join(' '));
+        _pino.error(msg);
+        broadcastLog({ level: 'error', msg, ts: Date.now() });
+        _writeLogLine('error', msg);
     },
     setLevel: (level) => {
         _pino.level = level;
     },
 };
+
+// Secrets store (roadmap A5): read once at startup; the UI/CLI keep it current afterwards.
+{
+    const st = secrets.load();
+    if (st.status === 'locked' || st.status === 'error') log.warn('secrets: ' + st.error + ' — she.secrets.get() returns undefined until the key is available');
+    else if (st.groups > 0) log.info(`secrets: ${st.groups} group(s) loaded`);
+}
 const pkg = require('../package.json');
 
 /**
@@ -362,6 +380,9 @@ function startOnce(reason) {
 // Wire up the MQTT API: pass the state store and a getter for the live MQTT client.
 // The getter always returns the current value of `mqtt` (null until connected).
 require('./web/mqtt-api').init(store, () => mqtt);
+require('./web/services-api').init(store, () => mqtt, {
+    getMqttConfig: () => ({ url: config.url, username: config.mqttUsername, password: config.mqttPassword }),
+});
 require('./web/ai-api').init(store);
 
 // MQTT message rate counter — reset on each stats poll
@@ -1182,6 +1203,19 @@ function runScript(script, name, _origin) {
         latitude: config.latitude,
         longitude: config.longitude,
     });
+    // she.secrets (roadmap A5): read-only access to the secrets store; unknown paths warn once per script
+    const _secretWarned = new Set();
+    she.secrets = Object.freeze({
+        get(p) {
+            const v = secrets.get(p);
+            if (v === undefined && !_secretWarned.has(String(p))) {
+                _secretWarned.add(String(p));
+                log.warn(logLabel, `secret "${p}" not found` + (secrets.status().status === 'locked' ? ' (secrets store is locked)' : ''));
+            }
+            return v;
+        },
+        has: (p) => secrets.has(p),
+    });
     // she.setTimeout / she.clearTimeout — tracked versions for use by stdlib and
     // sandbox modules that don't have direct access to the Sandbox context.
     she.setTimeout = (fn, delay, ...args) => {

package/src/lib/npm-registry.js

@@ -0,0 +1,80 @@
+'use strict';
+
+/**
+ * Small npm registry client with a 24 h cache — "is there a newer version of
+ * <package>?" for the Services page (roadmap I4) and, later, the adapter
+ * catalog (I7). Best-effort: every failure yields null, nothing throws.
+ */
+
+const semverCompare = require('semantic-compare');
+
+const REGISTRY = 'https://registry.npmjs.org';
+const TTL = 24 * 60 * 60 * 1000;
+
+/** @type {Map<string, {version: string|null, ts: number, promise?: Promise<string|null>}>} */
+const _cache = new Map();
+
+let _fetch = (...args) => fetch(...args);
+
+/** Test hook: replace the fetch implementation. */
+function setFetch(fn) {
+    _fetch = fn;
+}
+
+function clearCache() {
+    _cache.clear();
+}
+
+function validName(name) {
+    return typeof name === 'string' && /^(@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*$/.test(name);
+}
+
+/**
+ * Latest published version of a package (dist-tag `latest`), cached for 24 h.
+ * @param {string} name
+ * @param {{force?: boolean, now?: number}} [opts]
+ * @returns {Promise<string|null>}
+ */
+async function latestVersion(name, opts = {}) {
+    if (!validName(name)) return null;
+    const now = opts.now ?? Date.now();
+    const hit = _cache.get(name);
+    if (hit && !opts.force) {
+        if (hit.promise) return hit.promise;
+        if (now - hit.ts < TTL) return hit.version;
+    }
+    const promise = (async () => {
+        try {
+            const res = await _fetch(`${REGISTRY}/${name}/latest`, { signal: AbortSignal.timeout(10000) });
+            if (!res.ok) throw new Error(String(res.status));
+            const data = await res.json();
+            return typeof data.version === 'string' ? data.version : null;
+        } catch {
+            return hit ? hit.version : null; // keep the last good answer
+        }
+    })();
+    _cache.set(name, { version: hit ? hit.version : null, ts: now, promise });
+    const version = await promise;
+    _cache.set(name, { version, ts: now });
+    return version;
+}
+
+/**
+ * Newer version available? null when unknown.
+ * @param {string} name
+ * @param {string|null} installed
+ * @returns {Promise<{latest: string|null, updateAvailable: boolean|null}>}
+ */
+async function updateInfo(name, installed) {
+    const latest = await latestVersion(name);
+    if (!latest || !installed) return { latest, updateAvailable: null };
+    let updateAvailable = null;
+    try {
+        updateAvailable = semverCompare(installed, latest) < 0;
+    } catch {
+        /* unparsable version */
+    }
+    return { latest, updateAvailable };
+}
+
+module.exports = { latestVersion, updateInfo, validName, setFetch, clearCache, REGISTRY };

package/src/lib/secrets.js

@@ -0,0 +1,320 @@
+'use strict';
+
+/**
+ * Secrets store (roadmap A5): named groups of string fields, kept in an AES-256-GCM encrypted
+ * file under the config dir. Values never leave the daemon — the HTTP API is write-only, the only
+ * reader is script code via she.secrets — except *plain* fields (a username, a host): those are
+ * listed in clear, while *secret* fields are write-only; marking a field secret is one-way (A9).
+ * The key comes from SHE_SECRETS_KEY (hex or base64,
+ * 32 bytes) or from a key file next to the data, generated 0600 on first write.
+ *
+ * The point of the encryption is that a backup or a copied ~/.she holds no plaintext; the key
+ * file next to the data protects nothing against a local reader — see doc/security.md.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const { CONFIG_ROOT } = require('./storage');
+
+const NAME_RE = /^[A-Za-z0-9_.-]{1,64}$/;
+const MAX_VALUE = 64 * 1024;
+const MIN_REDACT = 6;
+const ALG = 'aes-256-gcm';
+
+let _file = path.join(CONFIG_ROOT, 'secrets.enc');
+let _keyFile = path.join(CONFIG_ROOT, 'secrets.key');
+let _env = process.env;
+
+/** @type {Record<string, Record<string, {value: string, changed: number}>>} */
+let _data = {};
+let _status = 'empty'; // empty | ok | locked | error
+let _error = null;
+let _keySource = null; // env | file | null
+let _redactList = [];
+
+/** Test hook: point the store at other files / another environment and forget everything. */
+function init({ file, keyFile, env } = {}) {
+    if (file) _file = file;
+    if (keyFile) _keyFile = keyFile;
+    if (env) _env = env;
+    _data = {};
+    _status = 'empty';
+    _error = null;
+    _keySource = null;
+    _redactList = [];
+}
+
+/** Entries written before kinds existed carry no flag and were secrets by definition. */
+function isSecret(e) {
+    return e.secret !== false;
+}
+
+function validName(n) {
+    return typeof n === 'string' && NAME_RE.test(n);
+}
+
+function parseKey(str, what) {
+    const s = String(str).trim();
+    let buf;
+    if (/^[0-9a-fA-F]{64}$/.test(s)) buf = Buffer.from(s, 'hex');
+    else buf = Buffer.from(s, 'base64');
+    if (buf.length !== 32) throw new Error(`${what} must be a 32-byte key as hex (64 chars) or base64`);
+    return buf;
+}
+
+/** The encryption key, or null when there is none and `create` is false. */
+function loadKey({ create = false } = {}) {
+    if (_env.SHE_SECRETS_KEY) {
+        _keySource = 'env';
+        return parseKey(_env.SHE_SECRETS_KEY, 'SHE_SECRETS_KEY');
+    }
+    try {
+        const key = parseKey(fs.readFileSync(_keyFile, 'utf8'), _keyFile);
+        _keySource = 'file';
+        return key;
+    } catch (err) {
+        if (err.code !== 'ENOENT') throw err;
+    }
+    if (!create) {
+        _keySource = null;
+        return null;
+    }
+    const key = crypto.randomBytes(32);
+    fs.mkdirSync(path.dirname(_keyFile), { recursive: true });
+    fs.writeFileSync(_keyFile, key.toString('hex') + '\n', { mode: 0o600 });
+    _keySource = 'file';
+    return key;
+}
+
+function encrypt(key, plaintext) {
+    const iv = crypto.randomBytes(12);
+    const c = crypto.createCipheriv(ALG, key, iv);
+    const data = Buffer.concat([c.update(plaintext, 'utf8'), c.final()]);
+    return { v: 1, alg: ALG, iv: iv.toString('base64'), tag: c.getAuthTag().toString('base64'), data: data.toString('base64') };
+}
+
+function decrypt(key, obj) {
+    const d = crypto.createDecipheriv(ALG, key, Buffer.from(obj.iv, 'base64'));
+    d.setAuthTag(Buffer.from(obj.tag, 'base64'));
+    return Buffer.concat([d.update(Buffer.from(obj.data, 'base64')), d.final()]).toString('utf8');
+}
+
+function rebuildRedact() {
+    const vals = new Set();
+    for (const g of Object.values(_data)) for (const f of Object.values(g)) if (isSecret(f) && f.value.length >= MIN_REDACT) vals.add(f.value);
+    _redactList = [...vals].sort((a, b) => b.length - a.length);
+}
+
+/**
+ * Read the file. Never creates anything. Returns the status.
+ * @returns {{status: string, error: string|null, groups: number}}
+ */
+function load() {
+    _data = {};
+    _error = null;
+    let raw;
+    try {
+        raw = fs.readFileSync(_file, 'utf8');
+    } catch (err) {
+        if (err.code === 'ENOENT') {
+            _status = 'empty';
+            try {
+                loadKey();
+            } catch (e) {
+                _status = 'error';
+                _error = e.message;
+            }
+            rebuildRedact();
+            return status();
+        }
+        _status = 'error';
+        _error = err.message;
+        return status();
+    }
+    let key;
+    try {
+        key = loadKey();
+    } catch (err) {
+        _status = 'error';
+        _error = err.message;
+        return status();
+    }
+    if (!key) {
+        _status = 'locked';
+        _error = `no key: set SHE_SECRETS_KEY or restore ${_keyFile}`;
+        return status();
+    }
+    try {
+        const obj = JSON.parse(raw);
+        if (!obj || obj.alg !== ALG) throw new Error('unknown file format');
+        const parsed = JSON.parse(decrypt(key, obj));
+        _data = parsed && typeof parsed === 'object' ? parsed : {};
+        _status = 'ok';
+    } catch (err) {
+        _status = 'locked';
+        _error = /unable to authenticate|auth/i.test(err.message) ? 'wrong key: the file was encrypted with another key' : 'cannot read the file: ' + err.message;
+    }
+    rebuildRedact();
+    return status();
+}
+
+function save() {
+    const key = loadKey({ create: true });
+    fs.mkdirSync(path.dirname(_file), { recursive: true });
+    const tmp = _file + '.tmp';
+    fs.writeFileSync(tmp, JSON.stringify(encrypt(key, JSON.stringify(_data))) + '\n', { mode: 0o600 });
+    fs.renameSync(tmp, _file);
+    _status = 'ok';
+    _error = null;
+    rebuildRedact();
+}
+
+function assertWritable() {
+    if (_status === 'locked' || _status === 'error') {
+        const err = new Error('secrets store is locked: ' + _error);
+        err.code = 'LOCKED';
+        throw err;
+    }
+}
+
+/** Status for the UI / CLI: no values. */
+function status() {
+    return { status: _status, error: _error, keySource: _keySource, file: _file, keyFile: _keyFile, groups: Object.keys(_data).length };
+}
+
+/** Names and change times only. */
+function list() {
+    return Object.keys(_data)
+        .sort()
+        .map((name) => {
+            const fields = Object.keys(_data[name])
+                .sort()
+                .map((f) => {
+                    const e = _data[name][f];
+                    return isSecret(e) ? { name: f, changed: e.changed, secret: true } : { name: f, changed: e.changed, secret: false, value: e.value };
+                });
+            return { name, changed: Math.max(0, ...fields.map((f) => f.changed)), fields };
+        });
+}
+
+function splitPath(p) {
+    if (typeof p !== 'string') return null;
+    const parts = p.split('/');
+    if (parts.length > 2 || !parts.every(validName)) return null;
+    return parts;
+}
+
+/** 'group/field' → string, 'group' → frozen {field: value}; undefined when absent or locked. */
+function get(p) {
+    const parts = splitPath(p);
+    if (!parts) return undefined;
+    const g = _data[parts[0]];
+    if (!g) return undefined;
+    if (parts.length === 1) {
+        const out = {};
+        for (const [f, v] of Object.entries(g)) out[f] = v.value;
+        return Object.freeze(out);
+    }
+    return g[parts[1]] ? g[parts[1]].value : undefined;
+}
+
+function has(p) {
+    return get(p) !== undefined;
+}
+
+/**
+ * Create or replace a field. `secret` decides the kind of a *new* field (default: secret);
+ * an existing secret field stays secret whatever is passed — the only way down is delete.
+ */
+function set(group, field, value, opts = {}) {
+    const now = typeof opts === 'number' ? opts : (opts.now ?? Date.now());
+    assertWritable();
+    if (!validName(group) || !validName(field)) throw Object.assign(new Error('names: letters, digits, _ . - (max 64)'), { code: 'INVALID_NAME' });
+    if (typeof value !== 'string' || value.length === 0) throw Object.assign(new Error('value must be a non-empty string'), { code: 'INVALID_VALUE' });
+    if (Buffer.byteLength(value) > MAX_VALUE) throw Object.assign(new Error(`value exceeds ${MAX_VALUE} bytes`), { code: 'INVALID_VALUE' });
+    if (!_data[group]) _data[group] = {};
+    const existing = _data[group][field];
+    const secret = existing ? isSecret(existing) || opts.secret === true : opts.secret !== false;
+    _data[group][field] = { value, changed: now, secret };
+    save();
+    return { group, field, changed: now, secret };
+}
+
+/** Mark a plain field secret — one-way; returns false when the field does not exist. */
+function mark(group, field) {
+    assertWritable();
+    if (!validName(group) || !validName(field)) throw Object.assign(new Error('names: letters, digits, _ . - (max 64)'), { code: 'INVALID_NAME' });
+    const e = _data[group] && _data[group][field];
+    if (!e) return false;
+    if (!isSecret(e)) {
+        e.secret = true;
+        save();
+    }
+    return true;
+}
+
+/** Remove a field, or a whole group when `field` is omitted. Returns false when nothing was there. */
+function remove(group, field) {
+    assertWritable();
+    if (!validName(group) || (field !== undefined && !validName(field))) throw Object.assign(new Error('names: letters, digits, _ . - (max 64)'), { code: 'INVALID_NAME' });
+    const g = _data[group];
+    if (!g) return false;
+    if (field === undefined) delete _data[group];
+    else {
+        if (!g[field]) return false;
+        delete g[field];
+        if (Object.keys(g).length === 0) delete _data[group];
+    }
+    save();
+    return true;
+}
+
+/** Replace every known secret value (6+ chars) in a log line with ***. */
+function redact(str) {
+    if (_redactList.length === 0 || typeof str !== 'string') return str;
+    let out = str;
+    for (const v of _redactList) if (out.includes(v)) out = out.split(v).join('***');
+    return out;
+}
+
+/**
+ * CLI: she --secret-set group/field [--plain] (value on stdin) | --secret-delete group[/field] | --secret-list.
+ * Returns the exit code.
+ */
+function cli(argv, io = { stdout: process.stdout, stderr: process.stderr, stdin: () => fs.readFileSync(0, 'utf8') }) {
+    const [cmd, arg, flag] = argv;
+    const st = load();
+    const fail = (m) => {
+        io.stderr.write('she: ' + m + '\n');
+        return 1;
+    };
+    try {
+        if (cmd === '--secret-list') {
+            if (st.status === 'locked' || st.status === 'error') return fail('secrets store is locked: ' + st.error);
+            for (const g of list())
+                for (const f of g.fields) io.stdout.write(`${g.name}/${f.name}\t${new Date(f.changed).toISOString()}\t${f.secret ? 'secret' : 'plain: ' + f.value}\n`);
+            return 0;
+        }
+        if (cmd === '--secret-set') {
+            const parts = arg && splitPath(arg);
+            if (!parts || parts.length !== 2 || (flag !== undefined && flag !== '--plain')) return fail('usage: she --secret-set <group>/<field> [--plain]  (value on stdin)');
+            const value = String(io.stdin()).replace(/\r?\n$/, '');
+            const r = set(parts[0], parts[1], value, { secret: flag !== '--plain' });
+            io.stdout.write(`set ${r.group}/${r.field} (${r.secret ? 'secret' : 'plain'})\n`);
+            return 0;
+        }
+        if (cmd === '--secret-delete') {
+            const parts = arg && splitPath(arg);
+            if (!parts) return fail('usage: she --secret-delete <group>[/<field>]');
+            if (!remove(parts[0], parts[1])) return fail(`no such secret: ${arg}`);
+            io.stdout.write(`deleted ${arg}\n`);
+            return 0;
+        }
+        return fail('unknown secrets command: ' + cmd);
+    } catch (err) {
+        return fail(err.message);
+    }
+}
+
+module.exports = { init, load, status, list, get, has, set, mark, remove, redact, cli, validName, NAME_RE, MAX_VALUE, MIN_REDACT };

package/src/lib/services-catalog.js

@@ -0,0 +1,144 @@
+'use strict';
+
+/**
+ * Adapter catalog (roadmap I7): the npm packages of the publishers the user trusts whose latest
+ * version depends on mqtt-interfaces-core. No keyword, no registration — depending on the core
+ * is the membership criterion.
+ *
+ *   1. registry search `maintainer:<publisher>` (official, one request per publisher)
+ *   2. abbreviated packument per hit (`application/vnd.npm.install-v1+json`, small) → keep the
+ *      package when `versions[dist-tags.latest].dependencies` has mqtt-interfaces-core
+ *   3. full packument for the members only → description, homepage, repository, `mqttInterfaces`
+ *
+ * Cached for 24 h with the last good result kept on failure; `refresh` forces a new sweep.
+ * Best effort throughout: a publisher that fails is reported in `errors`, the rest still lists.
+ */
+
+const CORE = 'mqtt-interfaces-core';
+const REGISTRY = 'https://registry.npmjs.org';
+const TTL = 24 * 60 * 60 * 1000;
+
+let _fetch = (...args) => fetch(...args);
+function setFetch(fn) {
+    _fetch = fn;
+}
+
+/** @type {{key: string, ts: number, result: object, promise?: Promise<object>} | null} */
+let _cache = null;
+function clearCache() {
+    _cache = null;
+}
+
+function validPublisher(p) {
+    return typeof p === 'string' && /^[a-z0-9][a-z0-9._-]{0,63}$/i.test(p);
+}
+
+async function getJson(url, headers = {}) {
+    const res = await _fetch(url, { headers: { accept: 'application/json', ...headers }, signal: AbortSignal.timeout(15000) });
+    if (!res.ok) throw new Error(`${res.status} ${url}`);
+    return res.json();
+}
+
+/** Package names a publisher maintains (registry search; 250 is the API's page maximum). */
+async function packagesOf(publisher) {
+    const data = await getJson(`${REGISTRY}/-/v1/search?text=${encodeURIComponent('maintainer:' + publisher)}&size=250`);
+    return (data.objects || [])
+        .map((o) => o.package)
+        .filter((p) => p && typeof p.name === 'string' && p.name !== CORE && !p.name.startsWith('@'))
+        .map((p) => ({ name: p.name, version: p.version, description: p.description || '', date: p.date || null, publisher: (p.publisher && p.publisher.username) || publisher }));
+}
+
+/** Does the latest version depend on the core? Returns the range or null. */
+async function coreDependency(name) {
+    const data = await getJson(`${REGISTRY}/${name}`, { accept: 'application/vnd.npm.install-v1+json' });
+    const latest = data['dist-tags'] && data['dist-tags'].latest;
+    const v = latest && data.versions && data.versions[latest];
+    if (!v) return null;
+    const range = (v.dependencies && v.dependencies[CORE]) || null;
+    return range ? { range, version: latest } : null;
+}
+
+/** Details of a member from the full packument. */
+async function details(name, version) {
+    const data = await getJson(`${REGISTRY}/${name}`);
+    const v = (data.versions && data.versions[version]) || {};
+    const repo = typeof data.repository === 'object' && data.repository ? data.repository.url : data.repository;
+    return {
+        description: v.description || data.description || '',
+        homepage: v.homepage || data.homepage || null,
+        repository: typeof repo === 'string' ? repo.replace(/^git\+/, '').replace(/\.git$/, '') : null,
+        mqttInterfaces: v.mqttInterfaces && typeof v.mqttInterfaces === 'object' ? v.mqttInterfaces : null,
+        deprecated: typeof v.deprecated === 'string' ? v.deprecated : null,
+        maintainers: Array.isArray(data.maintainers) ? data.maintainers.map((m) => m.name).filter(Boolean) : [],
+        published: data.time && data.time[version] ? data.time[version] : null,
+    };
+}
+
+/**
+ * Sweep the registry for the trusted publishers' adapters.
+ * @param {string[]} publishers
+ * @returns {Promise<{packages: object[], publishers: string[], errors: object[], fetchedAt: number}>}
+ */
+async function sweep(publishers, now) {
+    const errors = [];
+    const seen = new Map();
+    for (const publisher of publishers) {
+        let list;
+        try {
+            list = await packagesOf(publisher);
+        } catch (err) {
+            errors.push({ publisher, error: err.message });
+            continue;
+        }
+        for (const p of list) {
+            if (seen.has(p.name)) continue;
+            try {
+                const dep = await coreDependency(p.name);
+                if (!dep) continue;
+                const d = await details(p.name, dep.version);
+                if (d.deprecated) continue;
+                seen.set(p.name, { name: p.name, version: dep.version, coreRange: dep.range, publisher, ...d });
+            } catch (err) {
+                errors.push({ package: p.name, error: err.message });
+            }
+        }
+    }
+    const packages = [...seen.values()].sort((a, b) => a.name.localeCompare(b.name));
+    return { packages, publishers, errors, fetchedAt: now };
+}
+
+/**
+ * The catalog for the given trusted publishers (cached).
+ * @param {string[]} publishers
+ * @param {{force?: boolean, now?: number}} [opts]
+ */
+async function catalog(publishers, opts = {}) {
+    const list = [...new Set((publishers || []).filter(validPublisher).map((p) => p.toLowerCase()))];
+    const now = opts.now ?? Date.now();
+    const key = list.join(',');
+    if (list.length === 0) return { packages: [], publishers: [], errors: [], fetchedAt: now, cached: false };
+    if (_cache && _cache.key === key && !opts.force) {
+        if (_cache.promise) return { ...(await _cache.promise), cached: true };
+        if (now - _cache.ts < TTL) return { ..._cache.result, cached: true };
+    }
+    const previous = _cache && _cache.key === key ? _cache.result : null;
+    const promise = sweep(list, now).then((r) => {
+        // keep the last good list when the whole sweep produced nothing but errors
+        if (r.packages.length === 0 && r.errors.length > 0 && previous && previous.packages.length > 0) {
+            return { ...previous, errors: r.errors, stale: true };
+        }
+        return r;
+    });
+    _cache = { key, ts: now, result: previous, promise };
+    try {
+        const result = await promise;
+        _cache = { key, ts: now, result };
+        return { ...result, cached: false };
+    } catch (err) {
+        _cache = previous ? { key, ts: now, result: previous } : null;
+        if (previous) return { ...previous, errors: [{ error: err.message }], stale: true, cached: true };
+        throw err;
+    }
+}
+
+module.exports = { catalog, sweep, packagesOf, coreDependency, details, validPublisher, setFetch, clearCache, CORE, REGISTRY };