smart-home-engine 1.51.3 → 1.52.0

package/src/web/ai-context.js

@@ -1,92 +1,136 @@
 'use strict';
 
 /**
- * AI context builder — assembles the full system prompt from per-request flags,
- * the current editor context (script or DB view/document), and live data
- * from the running daemon (MQTT state, sheDB, Matter devices).
+ * AI context builder (roadmap I25) — assembles the system prompt from sections:
  *
- * Kept separate from ai-api.js so prompt files and context logic can evolve
- * independently of the HTTP routing / provider adapter code.
+ *   static (the same for every request of a mode and profile; cached by Anthropic, I22):
+ *     role · conventions · api reference (full or compact) · tools (when offered) · formats · compact steps
+ *   dynamic (per request):
+ *     the house (derived from the daemon: name, prefix, topic prefixes, adapters, the user's notes) ·
+ *     the current script / view / document · sheDB ids and samples · attachments
+ *
+ * Profiles: 'capable' for the current hosted models, 'compact' for small local ones — the facts are the same
+ * files, the compact profile gets the shorter reference, explicit steps and a budget. Nothing of an installation
+ * is in these files: the house section is computed at request time.
  */
 
 const fs = require('fs');
 const path = require('path');
 
-// Load prompt templates once at startup — plain Markdown files, no escaping needed
 const P = path.join(__dirname, 'prompts');
-const SCRIPTS_BASE_PROMPT = fs.readFileSync(path.join(P, 'scripts-base.md'), 'utf8').trim();
-const SHE_API_REF = fs.readFileSync(path.join(P, 'she-api-ref.md'), 'utf8').trim();
-const DB_VIEW_PROMPT = fs.readFileSync(path.join(P, 'db-view.md'), 'utf8').trim();
-const DB_DOC_PROMPT = fs.readFileSync(path.join(P, 'db-doc.md'), 'utf8').trim();
+const read = (n) => fs.readFileSync(path.join(P, n), 'utf8').trim();
+const ROLE = read('role.md');
+const CONVENTIONS = read('conventions.md');
+const API_REF = read('api-ref.md');
+const API_REF_COMPACT = read('api-ref-compact.md');
+const FORMATS = read('formats.md');
+const TOOLS = read('tools.md');
+const COMPACT_STEPS = read('compact.md');
+const DB_VIEW_PROMPT = read('db-view.md');
+const DB_DOC_PROMPT = read('db-doc.md');
+
+const DEFAULT_COMPACT_BUDGET = 24000; // characters, about 6k tokens: a small local model's comfortable share for the prompt
 
 /**
- * Build the full system prompt, including optional context sections.
- *
- * @param {object} requestCtx
- *   { apiref, mqtt, shedb, matter, sampleDocs }
- * @param {{ path?: string, content?: string } | null} currentScript
- * @param {{ id?: string, filter?: string, map?: string, reduce?: string } | null} currentView
- * @param {{ id?: string, content?: string } | null} currentDoc
- * @param {import('../lib/state-store') | null} store
- * @returns {string}
+ * The derived house section: what this installation looks like, from the running daemon — never from a file.
+ * @param {{ name?: string, variablePrefix?: string, version?: string } | null} cfg
+ * @param {Iterable<[string, object]> | null} entries  the state store's mqtt entries
+ * @param {Array<{instance: string, adapter?: string|null, connected?: number|null}>} instances
+ * @param {Array<{text: string}>} notes  the user's confirmed facts (roadmap I29)
  */
-function buildSystemPrompt(requestCtx, currentScript, currentView, currentDoc, store, extraFiles) {
-    const isViewMode = !!currentView?.id;
-    const isDocMode = !!currentDoc?.id;
-
-    let basePrompt;
-    if (isViewMode) {
-        basePrompt = DB_VIEW_PROMPT;
-    } else if (isDocMode) {
-        basePrompt = DB_DOC_PROMPT;
-    } else {
-        basePrompt = SCRIPTS_BASE_PROMPT;
+function houseSection(cfg, entries, instances, notes) {
+    const lines = ['## This installation'];
+    if (cfg?.name)
+        lines.push(
+            `- the daemon's MQTT name is \`${cfg.name}\`${cfg.version ? ` (she ${cfg.version})` : ''}; its variables live under \`${cfg.variablePrefix || 'var'}/status/…\` and \`${cfg.variablePrefix || 'var'}/set/…\``,
+        );
+    if (entries) {
+        const counts = new Map();
+        let total = 0;
+        for (const [topic] of entries) {
+            total++;
+            const first = topic.split('/')[0];
+            counts.set(first, (counts.get(first) || 0) + 1);
+        }
+        const top = [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 15);
+        if (total) lines.push(`- about ${roundish(total)} topics on the broker; the largest prefixes: ${top.map(([k, v]) => `\`${k}/\` (${roundish(v)})`).join(', ')}`);
+    }
+    const known = (instances || []).filter((i) => i.adapter).map((i) => `\`${i.instance}/\` ${i.adapter}${i.connected === 0 ? ' (offline)' : ''}`);
+    if (known.length) lines.push(`- adapter instances: ${known.slice(0, 20).join(', ')}${known.length > 20 ? ` … (${known.length})` : ''}`);
+    if (notes && notes.length) {
+        lines.push('- facts the user confirmed earlier (trust them over guesses):');
+        for (const n of notes.slice(0, 300)) lines.push(`  - [${n.id}] ${String(n.text).replace(/\s+/g, ' ').slice(0, 200)}`);
     }
+    return lines.length > 1 ? lines.join('\n') : '';
+}
+
+/** 8702 → "8700", 143 → "140", 12 → "12": stable across requests, so the cached prefix is not disturbed by every message */
+function roundish(n) {
+    if (n < 20) return String(n);
+    const m = Math.pow(10, Math.floor(Math.log10(n)) - 1);
+    return String(Math.round(n / m) * m);
+}
+
+/**
+ * Build the full system prompt as one string (the OpenAI-compatible providers, the /prompt route).
+ */
+function buildSystemPrompt(requestCtx, currentScript, currentView, currentDoc, store, extraFiles, opts) {
+    const { staticText, dynamicText } = buildSystemPromptParts(requestCtx, currentScript, currentView, currentDoc, store, extraFiles, opts);
+    return dynamicText ? staticText + '\n\n' + dynamicText : staticText;
+}
 
-    const parts = [basePrompt];
+/**
+ * The prompt in its static and dynamic part.
+ * @param {object} requestCtx  { apiref, tools, shedb, sampleDocs }
+ * @param {{ profile?: 'capable'|'compact', toolsOffered?: boolean, budgetChars?: number, house?: string }} [opts]
+ * @returns {{ staticText: string, dynamicText: string, dropped: string[] }}
+ */
+function buildSystemPromptParts(requestCtx, currentScript, currentView, currentDoc, store, extraFiles, opts = {}) {
+    const profile = opts.profile === 'compact' ? 'compact' : 'capable';
+    const isViewMode = !!currentView?.id;
+    const isDocMode = !!currentDoc?.id;
+    const toolsOffered = opts.toolsOffered !== undefined ? !!opts.toolsOffered : !!requestCtx.tools;
 
-    if (requestCtx.apiref && !isViewMode && !isDocMode) {
-        parts.push(SHE_API_REF);
+    const staticParts = [];
+    if (isViewMode) staticParts.push(DB_VIEW_PROMPT);
+    else if (isDocMode) staticParts.push(DB_DOC_PROMPT);
+    else {
+        staticParts.push(ROLE, CONVENTIONS);
+        if (requestCtx.apiref !== false) staticParts.push(profile === 'compact' ? API_REF_COMPACT : API_REF);
+        if (toolsOffered) staticParts.push(TOOLS);
+        staticParts.push(FORMATS);
+        if (profile === 'compact') staticParts.push(COMPACT_STEPS);
     }
 
+    // dynamic sections, each with a name so the budget can drop the least important first
+    const sections = [];
+    if (opts.house) sections.push({ name: 'house', text: opts.house });
     if (currentScript?.path && typeof currentScript.content === 'string') {
-        parts.push(`## Current script: ${currentScript.path}\n\`\`\`javascript\n${currentScript.content}\n\`\`\``);
+        sections.push({ name: 'script', text: `## Current script: ${currentScript.path}\n\`\`\`javascript\n${currentScript.content}\n\`\`\`` });
     }
-
     if (currentView?.id) {
         const filterStr = (currentView.filter || '').trim();
         const mapBody = (currentView.map || '').trim();
         const reduceBody = (currentView.reduce || '').trim();
-        const viewLines = [`## Current view: ${currentView.id}`];
-        viewLines.push(`Filter: ${filterStr || '(none)'}`);
-        viewLines.push(`Map:\n\`\`\`javascript\n${mapBody || '// (empty)'}\n\`\`\``);
-        if (reduceBody) {
-            viewLines.push(`Reduce:\n\`\`\`javascript\n${reduceBody}\n\`\`\``);
-        } else {
-            viewLines.push('Reduce: (none)');
-        }
-        parts.push(viewLines.join('\n'));
+        const viewLines = [`## Current view: ${currentView.id}`, `Filter: ${filterStr || '(none)'}`, `Map:\n\`\`\`javascript\n${mapBody || '// (empty)'}\n\`\`\``];
+        viewLines.push(reduceBody ? `Reduce:\n\`\`\`javascript\n${reduceBody}\n\`\`\`` : 'Reduce: (none)');
+        sections.push({ name: 'view', text: viewLines.join('\n') });
     }
-
     if (currentDoc?.id) {
         const content = typeof currentDoc.content === 'string' ? currentDoc.content : JSON.stringify(currentDoc.content, null, 2);
-        parts.push(`## Current document: ${currentDoc.id}\n\`\`\`json\n${content}\n\`\`\``);
+        sections.push({ name: 'doc', text: `## Current document: ${currentDoc.id}\n\`\`\`json\n${content}\n\`\`\`` });
     }
-
     if (requestCtx.shedb) {
         try {
             const core = require('./shedb').getCore();
             if (core) {
                 const ids = Object.keys(core.docs).sort();
-                if (ids.length > 0) {
-                    parts.push(`## sheDB document IDs (${ids.length} total)\n${ids.slice(0, 200).join('\n')}`);
-                }
+                if (ids.length > 0) sections.push({ name: 'shedb-ids', text: `## sheDB document IDs (${ids.length} total)\n${ids.slice(0, 200).join('\n')}` });
             }
         } catch {
-            // shedb not initialised — skip silently
+            /* shedb not initialised */
         }
     }
-
     if (requestCtx.sampleDocs) {
         try {
             const core = require('./shedb').getCore();
@@ -94,23 +138,90 @@ function buildSystemPrompt(requestCtx, currentScript, currentView, currentDoc, s
                 const ids = Object.keys(core.docs).sort().slice(0, 10);
                 if (ids.length > 0) {
                     const sample = ids.map((id) => `### ${id}\n${JSON.stringify(core.docs[id], null, 2)}`).join('\n\n');
-                    parts.push(`## Sample sheDB documents (${ids.length} shown)\n${sample}`);
+                    sections.push({ name: 'shedb-samples', text: `## Sample sheDB documents (${ids.length} shown)\n${sample}` });
                 }
             }
         } catch {
-            // shedb not initialised — skip silently
+            /* shedb not initialised */
         }
     }
+    for (const f of extraFiles || []) {
+        const ext = (f.name.match(/\.([^.]+)$/) || [])[1] || '';
+        sections.push({ name: 'file:' + f.name, text: `## Attached file: ${f.name}\n\`\`\`${ext.toLowerCase()}\n${f.content}\n\`\`\`` });
+    }
 
-    if (extraFiles && extraFiles.length > 0) {
-        for (const f of extraFiles) {
-            const ext = (f.name.match(/\.([^.]+)$/) || [])[1] || '';
-            const lang = ext.toLowerCase();
-            parts.push(`## Attached file: ${f.name}\n\`\`\`${lang}\n${f.content}\n\`\`\``);
+    // the budget (compact profile, or an explicit one): drop the least important sections first
+    const dropped = [];
+    const budget = opts.budgetChars || (profile === 'compact' ? DEFAULT_COMPACT_BUDGET : 0);
+    if (budget > 0) {
+        const staticLen = staticParts.join('\n\n').length;
+        const order = ['shedb-samples', 'shedb-ids', 'file:', 'script'];
+        const size = () => staticLen + sections.reduce((n, s) => n + s.text.length + 2, 0);
+        for (const kind of order) {
+            while (size() > budget) {
+                const idx = kind === 'file:' ? sections.map((s) => s.name).findLastIndex((n) => n.startsWith('file:')) : sections.findIndex((s) => s.name === kind);
+                if (idx === -1) break;
+                dropped.push(sections[idx].name);
+                sections.splice(idx, 1);
+            }
         }
+        if (dropped.length) sections.push({ name: 'dropped', text: `(Left out to fit the model's context: ${dropped.join(', ')}. Ask for a part of it if needed.)` });
     }
 
-    return parts.join('\n\n');
+    return { staticText: staticParts.join('\n\n'), dynamicText: sections.map((s) => s.text).join('\n\n'), dropped };
+}
+
+/** which prompt profile fits a provider entry; `profile` on the entry overrides */
+function profileFor(ai) {
+    if (ai?.profile === 'compact' || ai?.profile === 'capable') return ai.profile;
+    if (!ai) return 'capable';
+    if (ai.provider === 'ollama') return 'compact';
+    if (ai.provider === 'openai') {
+        const u = String(ai.baseUrl || '').toLowerCase();
+        if (/localhost|127\.0\.0\.1|\.lan\b|\.local\b|\.home\b|:1234\b|:11434\b|:8080\b/.test(u)) return 'compact';
+    }
+    return 'capable';
+}
+
+/** strip the <think>…</think> blocks a thinking model returns as text (compact profile) */
+function stripThinking(text) {
+    return String(text)
+        .replace(/<think>[\s\S]*?<\/think>\s*/gi, '')
+        .replace(/^<think>[\s\S]*$/i, '')
+        .trim();
+}
+
+/** a stateful filter for a token stream: tokens inside <think>…</think> are swallowed */
+function thinkFilter(onToken) {
+    let inside = false;
+    let buf = '';
+    return (token) => {
+        buf += token;
+        for (;;) {
+            if (inside) {
+                const end = buf.indexOf('</think>');
+                if (end === -1) {
+                    buf = buf.slice(-8); // keep a tail in case the tag is split
+                    return;
+                }
+                buf = buf.slice(end + 8).replace(/^\s+/, '');
+                inside = false;
+            } else {
+                const start = buf.indexOf('<think>');
+                if (start === -1) {
+                    // emit all but a possible partial tag at the end
+                    const keep = buf.lastIndexOf('<');
+                    const safe = keep === -1 || buf.length - keep > 7 ? buf : buf.slice(0, keep);
+                    if (safe) onToken(safe);
+                    buf = buf.slice(safe.length);
+                    return;
+                }
+                if (start > 0) onToken(buf.slice(0, start));
+                buf = buf.slice(start + 7);
+                inside = true;
+            }
+        }
+    };
 }
 
-module.exports = { buildSystemPrompt };
+module.exports = { buildSystemPrompt, buildSystemPromptParts, houseSection, profileFor, stripThinking, thinkFilter, DEFAULT_COMPACT_BUDGET };

package/src/web/ai-drafts.js

@@ -0,0 +1,68 @@
+'use strict';
+
+/**
+ * Script drafts the AI proposes (roadmap I16, decision D-1): written to `<data-dir>/ai/drafts/<id>.json`, shown in
+ * the chat as a diff with an Apply button; nothing lands in the scripts directory without that click. Drafts older
+ * than a week are pruned at daemon start.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+const KEEP_MS = 7 * 24 * 3600 * 1000;
+let _dir = null;
+
+function init(dir) {
+    _dir = dir;
+    try {
+        fs.mkdirSync(_dir, { recursive: true });
+        const now = Date.now();
+        for (const n of fs.readdirSync(_dir)) {
+            if (!n.endsWith('.json')) continue;
+            const f = path.join(_dir, n);
+            try {
+                if (now - fs.statSync(f).mtimeMs > KEEP_MS) fs.rmSync(f, { force: true });
+            } catch {
+                /* ignore */
+            }
+        }
+    } catch {
+        /* the directory is created on the first draft */
+    }
+}
+
+function file(id) {
+    if (!/^d[0-9a-f]{8}$/.test(id)) return null;
+    return path.join(_dir, id + '.json');
+}
+
+/** @returns {object} the draft */
+function create({ path: scriptPath, content, base, note }) {
+    if (!_dir) throw new Error('drafts not initialised');
+    fs.mkdirSync(_dir, { recursive: true });
+    const draft = { id: 'd' + crypto.randomBytes(4).toString('hex'), path: scriptPath, content, base: base ?? null, note: note || '', createdAt: Date.now(), status: 'open' };
+    fs.writeFileSync(file(draft.id), JSON.stringify(draft));
+    return draft;
+}
+
+function get(id) {
+    const f = _dir && file(id);
+    if (!f) return null;
+    try {
+        return JSON.parse(fs.readFileSync(f, 'utf8'));
+    } catch {
+        return null;
+    }
+}
+
+function setStatus(id, status) {
+    const d = get(id);
+    if (!d) return null;
+    d.status = status;
+    d.updatedAt = Date.now();
+    fs.writeFileSync(file(id), JSON.stringify(d));
+    return d;
+}
+
+module.exports = { init, create, get, setStatus };

package/src/web/ai-memory.js

@@ -0,0 +1,106 @@
+'use strict';
+
+/**
+ * The AI chat's memory (roadmap I29, decision D-8): facts the user confirmed, one store per instance in the data
+ * directory — `<data-dir>/ai/memory.json`, next to the conversations. Never in the scripts directory (the git
+ * auto-commit) and never in a repository. Shown and editable on the AI page; the model may add and remove notes.
+ * Every note runs through the secrets redaction, and anything that looks like a key or password is refused.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+const MAX_NOTES = 300;
+const MAX_CHARS = 200;
+const SECRET_LIKE =
+    /(sk-[a-z0-9]{2,}-?[A-Za-z0-9_-]{8,}|password\s*[:=]|passwort\s*[:=]|api[_ -]?key\s*[:=]|token\s*[:=]|bearer\s+[A-Za-z0-9._-]{16,}|\b[A-Za-z0-9+/]{32,}={0,2}\b)/i;
+
+let _file = null;
+let _redact = (s) => s;
+let _notes = null;
+
+/**
+ * @param {string} dir  the ai directory in the data root
+ * @param {(s: string) => string} [redact]
+ */
+function init(dir, redact) {
+    _file = path.join(dir, 'memory.json');
+    if (redact) _redact = redact;
+    _notes = null;
+}
+
+function load() {
+    if (_notes) return _notes;
+    try {
+        const json = JSON.parse(fs.readFileSync(_file, 'utf8'));
+        _notes = Array.isArray(json.notes) ? json.notes.filter((n) => n && typeof n.id === 'string' && typeof n.text === 'string') : [];
+    } catch {
+        _notes = [];
+    }
+    return _notes;
+}
+
+function persist() {
+    if (!_file) throw new Error('memory not initialised');
+    fs.mkdirSync(path.dirname(_file), { recursive: true });
+    const tmp = _file + '.tmp';
+    fs.writeFileSync(tmp, JSON.stringify({ notes: _notes }, null, 2));
+    fs.renameSync(tmp, _file);
+}
+
+/** null when the text may be stored, otherwise the reason */
+function refusal(text) {
+    const t = String(text || '').trim();
+    if (!t) return 'empty note';
+    if (t.length > MAX_CHARS) return `a note has at most ${MAX_CHARS} characters (${t.length} given); split it or shorten it`;
+    if (SECRET_LIKE.test(t)) return 'this looks like a key, a token or a password; secrets belong in the Secrets tab, not in the memory';
+    return null;
+}
+
+function list() {
+    if (!_file) return [];
+    return load().map((n) => ({ ...n }));
+}
+
+/** @returns {{ note?: object, error?: string }} */
+function add(text, source = 'user') {
+    if (!_file) return { error: 'memory not initialised' };
+    const why = refusal(text);
+    if (why) return { error: why };
+    const notes = load();
+    const clean = _redact(String(text).trim().replace(/\s+/g, ' '));
+    const dup = notes.find((n) => n.text.toLowerCase() === clean.toLowerCase());
+    if (dup) return { note: { ...dup }, duplicate: true };
+    if (notes.length >= MAX_NOTES) return { error: `the memory holds at most ${MAX_NOTES} notes; forget one first` };
+    const note = { id: 'n' + crypto.randomBytes(4).toString('hex'), text: clean, createdAt: Date.now(), source: source === 'model' ? 'model' : 'user' };
+    notes.push(note);
+    persist();
+    return { note: { ...note } };
+}
+
+function update(id, text) {
+    if (!_file) return { error: 'memory not initialised' };
+    const why = refusal(text);
+    if (why) return { error: why };
+    const notes = load();
+    const note = notes.find((n) => n.id === id);
+    if (!note) return { error: `no note ${id}` };
+    note.text = _redact(String(text).trim().replace(/\s+/g, ' '));
+    note.source = 'user'; // edited by hand
+    note.updatedAt = Date.now();
+    persist();
+    return { note: { ...note } };
+}
+
+function remove(id) {
+    if (!_file) return { error: 'memory not initialised' };
+    const notes = load();
+    const idx = notes.findIndex((n) => n.id === id);
+    if (idx === -1) return { error: `no note ${id}` };
+    const [note] = notes.splice(idx, 1);
+    persist();
+    return { note };
+}
+
+module.exports = { init, list, add, update, remove, refusal, MAX_NOTES, MAX_CHARS };