smart-home-engine 0.18.0 → 0.18.1

package/src/lib/shedb-core.js

@@ -112,7 +112,10 @@ class SheDBCore extends EventEmitter {
 
         this._viewEnvs = {}; // id → { compileError? } — only syntax-check metadata
         this._saveTimer = null;
-        this._sendDbScheduled = false;
+        this._saveViewsTimer = null;
+        this._viewsPath = dbPath.endsWith('.json')
+            ? dbPath.slice(0, -5) + '-views.json'
+            : dbPath + '-views.json';
         this._worker = null;
 
         this._spawnWorker();
@@ -124,12 +127,13 @@ class SheDBCore extends EventEmitter {
     // -------------------------------------------------------------------------
 
     _load() {
+        let legacyViews = null;
         try {
             const data = JSON.parse(fs.readFileSync(this.dbPath, 'utf8'));
             this.rev = data.rev || 0;
             this.docs = data.docs || {};
             this.queries = data.queries || {};
-            this.views = data.views || {};
+            if (data.views && Object.keys(data.views).length > 0) legacyViews = data.views;
             this.log.info('shedb loaded ' + Object.keys(this.docs).length + ' docs, ' + Object.keys(this.queries).length + ' views from ' + this.dbPath);
         } catch (err) {
             if (err.code !== 'ENOENT') {
@@ -137,6 +141,30 @@ class SheDBCore extends EventEmitter {
             }
         }
 
+        try {
+            const vdata = JSON.parse(fs.readFileSync(this._viewsPath, 'utf8'));
+            this.views = vdata.views || {};
+        } catch (err) {
+            if (err.code !== 'ENOENT') {
+                this.log.warn('shedb: could not load views from ' + this._viewsPath + ': ' + err.message);
+            }
+            if (legacyViews) {
+                // Migrate: split legacy single-file format into two files
+                this.views = legacyViews;
+                this.log.info('shedb: migrating views to separate file ' + this._viewsPath);
+                try {
+                    const vtmp = this._viewsPath + '.tmp';
+                    fs.writeFileSync(vtmp, JSON.stringify({ views: this.views }), 'utf8');
+                    fs.renameSync(vtmp, this._viewsPath);
+                    const tmp = this.dbPath + '.tmp';
+                    fs.writeFileSync(tmp, JSON.stringify({ rev: this.rev, docs: this.docs, queries: this.queries }), 'utf8');
+                    fs.renameSync(tmp, this.dbPath);
+                } catch (e) {
+                    this.log.error('shedb: migration failed: ' + e.message);
+                }
+            }
+        }
+
         // Syntax-check persisted views and send state to worker
         for (const id of Object.keys(this.queries)) {
             this._syntaxCheck(id, this.queries[id]);
@@ -151,7 +179,7 @@ class SheDBCore extends EventEmitter {
         this._saveTimer = setTimeout(() => {
             const tmp = this.dbPath + '.tmp';
             try {
-                fs.writeFileSync(tmp, JSON.stringify({ rev: this.rev, docs: this.docs, queries: this.queries, views: this.views }), 'utf8');
+                fs.writeFileSync(tmp, JSON.stringify({ rev: this.rev, docs: this.docs, queries: this.queries }), 'utf8');
                 fs.renameSync(tmp, this.dbPath); // atomic on Linux (same FS)
             } catch (err) {
                 this.log.error('shedb: save failed: ' + err.message);
@@ -159,6 +187,19 @@ class SheDBCore extends EventEmitter {
         }, 250);
     }
 
+    _saveViews() {
+        clearTimeout(this._saveViewsTimer);
+        this._saveViewsTimer = setTimeout(() => {
+            const tmp = this._viewsPath + '.tmp';
+            try {
+                fs.writeFileSync(tmp, JSON.stringify({ views: this.views }), 'utf8');
+                fs.renameSync(tmp, this._viewsPath);
+            } catch (err) {
+                this.log.error('shedb: save views failed: ' + err.message);
+            }
+        }, 250);
+    }
+
     // -------------------------------------------------------------------------
     // Rev tracking — mirrors mqttDB pattern exactly
     // -------------------------------------------------------------------------
@@ -215,7 +256,7 @@ class SheDBCore extends EventEmitter {
             this.rev++;
             this._save();
             this.emit('update', id, this.docs[id]);
-            this._sendDb();
+            this._sendPatch(id, this.docs[id]);
             return true;
         }
         this._setRev(id, rev);
@@ -244,7 +285,7 @@ class SheDBCore extends EventEmitter {
             this.rev++;
             this._save();
             this.emit('update', id, this.docs[id]);
-            this._sendDb();
+            this._sendPatch(id, this.docs[id]);
             return true;
         }
         this._setRev(id, rev);
@@ -256,7 +297,7 @@ class SheDBCore extends EventEmitter {
         this.rev++;
         this._save();
         this.emit('update', id, '');
-        this._sendDb();
+        this._sendPatch(id, null);
     }
 
     /**
@@ -287,7 +328,7 @@ class SheDBCore extends EventEmitter {
             this.rev++;
             this._save();
             this.emit('update', id, this.docs[id]);
-            this._sendDb();
+            this._sendPatch(id, this.docs[id]);
             return true;
         }
         this._setRev(id, rev);
@@ -309,6 +350,7 @@ class SheDBCore extends EventEmitter {
             delete this._viewEnvs[id];
             delete this.views[id];
             this._save();
+            this._saveViews();
             if (this._worker) this._worker.postMessage({ type: 'delQuery', id });
             this.emit('view', id, '');
             return;
@@ -362,7 +404,7 @@ class SheDBCore extends EventEmitter {
                 delete this.views[id].result;
                 this.log.error('shedb view ' + id + ': ' + msg.error);
                 this.emit('view', id, this.views[id]);
-                this._save();
+                this._saveViews();
                 return;
             }
 
@@ -370,7 +412,7 @@ class SheDBCore extends EventEmitter {
                 this.views[id] = { _id: id, _rev: (prev._rev ?? -1) + 1, result: msg.result, length: msg.result.length };
                 delete this.views[id].error;
                 this.emit('view', id, this.views[id]);
-                this._save();
+                this._saveViews();
             }
         });
 
@@ -390,22 +432,20 @@ class SheDBCore extends EventEmitter {
         });
     }
 
-    /** Send full docs snapshot to the worker (debounced via setImmediate). */
-    _sendDb() {
-        if (this._sendDbScheduled) return;
-        this._sendDbScheduled = true;
-        setImmediate(() => {
-            this._sendDbScheduled = false;
-            if (this._worker) {
-                this._worker.postMessage({ type: 'db', docs: structuredClone(this.docs) });
-            }
-        });
+    /** Send a single-document patch (or deletion) to the worker — O(1). */
+    _sendPatch(id, doc) {
+        if (!this._worker) return;
+        if (doc) {
+            this._worker.postMessage({ type: 'patch', id, doc: structuredClone(doc) });
+        } else {
+            this._worker.postMessage({ type: 'del', id });
+        }
     }
 
-    /** Send full state (docs + all queries) to the worker — used on init and respawn. */
+    /** Send full state (docs + all queries) to the worker — used on init and respawn only. */
     _sendInitialState() {
         if (!this._worker) return;
-        this._worker.postMessage({ type: 'db', docs: structuredClone(this.docs) });
+        this._worker.postMessage({ type: 'init', docs: structuredClone(this.docs) });
         for (const id of Object.keys(this.queries)) {
             this._worker.postMessage({ type: 'query', id, payload: this.queries[id] });
         }

package/src/lib/shedb-worker.js

@@ -7,8 +7,10 @@
  *
  * Message protocol:
  *   Main → Worker:
- *     { type: 'db',       docs: {...} }              full docs snapshot
- *     { type: 'query',    id, payload: {filter?,map,reduce?} }
+ *     { type: 'init',  docs: {...} }              full snapshot (startup/respawn only)
+ *     { type: 'patch', id, doc }                  single document created/updated
+ *     { type: 'del',   id }                       single document deleted
+ *     { type: 'query', id, payload: {filter?,map,reduce?} }
  *     { type: 'delQuery', id }
  *
  *   Worker → Main:
@@ -25,7 +27,7 @@ const TIMEOUT = (workerData && workerData.scriptTimeout) || 5000;
 
 let docs = {};
 const queries = {};
-let queue = [];
+let queue = new Set();
 let running = false;
 
 // ---------------------------------------------------------------------------
@@ -37,22 +39,31 @@ function getProp(obj, propPath) {
     return propPath.split('.').reduce((cur, k) => (cur != null ? cur[k] : undefined), obj);
 }
 
+// Enqueue only the views whose filter matches the changed doc id.
+// Views with no filter must always re-run (they iterate all docs).
+function enqueueForDoc(docId) {
+    for (const [id, q] of Object.entries(queries)) {
+        if (!q.filter || mqttWildcard(docId, q.filter)) enqueue(id);
+    }
+}
+
 // ---------------------------------------------------------------------------
 // View queue
 // ---------------------------------------------------------------------------
 
 function enqueue(id) {
-    if (!queue.includes(id)) queue.push(id);
+    queue.add(id);
     if (!running) scheduleNext();
 }
 
 function scheduleNext() {
-    if (queue.length === 0) {
+    if (queue.size === 0) {
         running = false;
         return;
     }
     running = true;
-    const id = queue.shift();
+    const id = queue.values().next().value;
+    queue.delete(id);
     setImmediate(() => buildAndRun(id));
 }
 
@@ -63,20 +74,7 @@ function buildAndRun(id) {
         return;
     }
 
-    const { filter, map, reduce } = q;
-
-    // Build script source — same structure as the original in-process approach
-    let src = `api.map = function() {\n${map}\n};\napi._result = [];\n`;
-    if (filter) {
-        src += `api.forEachDocument(docId => { if (api.mqttWildcard(docId, ${JSON.stringify(filter)})) api.map.apply(api.getDocument(docId)); });\n`;
-    } else {
-        src += `api.forEachDocument(docId => { api.map.apply(api.getDocument(docId)); });\n`;
-    }
-    if (reduce) {
-        src += `api.reduce = function(result) {\n${reduce}\n};\napi._result = api.reduce(api._result);\n`;
-    }
-
-    // Sandbox — uses the local docs snapshot
+    // Sandbox — uses the live docs object (updated by patch/del messages)
     const sandbox = {
         api: {
             forEachDocument: (cb) => Object.keys(docs).forEach(cb),
@@ -89,9 +87,8 @@ function buildAndRun(id) {
     sandbox.emit = (item) => sandbox.api._result.push(item);
 
     try {
-        const script = new vm.Script(src, { filename: 'shedb-view-' + id });
         const ctx = vm.createContext(sandbox);
-        script.runInContext(ctx, { timeout: TIMEOUT });
+        q.script.runInContext(ctx, { timeout: TIMEOUT });
         parentPort.postMessage({ type: 'view', id, result: Array.from(ctx.api._result) });
     } catch (err) {
         parentPort.postMessage({ type: 'view', id, error: 'runtime: ' + err.message });
@@ -106,16 +103,44 @@ function buildAndRun(id) {
 
 parentPort.on('message', (msg) => {
     switch (msg.type) {
-        case 'db':
+        case 'init':
             docs = msg.docs;
-            // Re-run all registered views with the new docs snapshot
+            // Re-run all registered views with the fresh snapshot
             for (const id of Object.keys(queries)) enqueue(id);
             break;
 
-        case 'query':
-            queries[msg.id] = msg.payload;
+        case 'patch':
+            docs[msg.id] = msg.doc;
+            enqueueForDoc(msg.id);
+            break;
+
+        case 'del':
+            delete docs[msg.id];
+            enqueueForDoc(msg.id);
+            break;
+
+        case 'query': {
+            const { filter, map, reduce } = msg.payload;
+            let src = `api.map = function() {\n${map}\n};\napi._result = [];\n`;
+            if (filter) {
+                src += `api.forEachDocument(docId => { if (api.mqttWildcard(docId, ${JSON.stringify(filter)})) api.map.apply(api.getDocument(docId)); });\n`;
+            } else {
+                src += `api.forEachDocument(docId => { api.map.apply(api.getDocument(docId)); });\n`;
+            }
+            if (reduce) {
+                src += `api.reduce = function(result) {\n${reduce}\n};\napi._result = api.reduce(api._result);\n`;
+            }
+            let script;
+            try {
+                script = new vm.Script(src, { filename: 'shedb-view-' + msg.id });
+            } catch (err) {
+                parentPort.postMessage({ type: 'view', id: msg.id, error: 'compile: ' + err.message });
+                break;
+            }
+            queries[msg.id] = { ...msg.payload, script };
             enqueue(msg.id);
             break;
+        }
 
         case 'delQuery':
             delete queries[msg.id];

package/src/web/scripts-api.js

@@ -28,6 +28,11 @@ function hasShelibMarker(absDir) {
     return fs.existsSync(path.join(absDir, '.shelib'));
 }
 
+/** True if a .shedisable-<name> sibling exists for the given file or directory. */
+function hasShedisableMarker(abs) {
+    return fs.existsSync(path.join(path.dirname(abs), `.shedisable-${path.basename(abs)}`));
+}
+
 /** Flat list of all files with metadata and lib flag. */
 function walk(dir, base, parentIsLib) {
     let entries;
@@ -40,11 +45,13 @@ function walk(dir, base, parentIsLib) {
     const results = [];
     for (const entry of entries) {
         if (entry.name === '.shelib') continue;
+        if (entry.name.startsWith('.shedisable-')) continue;
         const rel = base ? `${base}/${entry.name}` : entry.name;
         if (entry.isDirectory()) {
             results.push(...walk(path.join(dir, entry.name), rel, lib));
         } else {
-            const stat = fs.statSync(path.join(dir, entry.name));
+            const abs = path.join(dir, entry.name);
+            const stat = fs.statSync(abs);
             results.push({ path: rel, size: stat.size, mtime: stat.mtimeMs, lib });
         }
     }
@@ -67,15 +74,19 @@ function buildTree(dir, base, parentIsLib) {
     const result = [];
     for (const entry of entries) {
         if (entry.name === '.shelib') continue;
+        if (entry.name.startsWith('.shedisable-')) continue;
         const rel = base ? `${base}/${entry.name}` : entry.name;
         const abs = path.join(dir, entry.name);
         if (entry.isDirectory()) {
             const childIsLib = lib || hasShelibMarker(abs);
+            const disabled = hasShedisableMarker(abs);
             const children = buildTree(abs, rel, childIsLib);
-            result.push({ type: 'dir', name: entry.name, path: rel, lib: childIsLib, children });
+            result.push({ type: 'dir', name: entry.name, path: rel, lib: childIsLib, disabled, children });
         } else {
             const stat = fs.statSync(abs);
-            result.push({ type: 'file', name: entry.name, path: rel, lib, size: stat.size, mtime: stat.mtimeMs });
+            const isJs = entry.name.endsWith('.js');
+            const disabled = isJs ? hasShedisableMarker(abs) : false;
+            result.push({ type: 'file', name: entry.name, path: rel, lib, size: stat.size, mtime: stat.mtimeMs, ...(isJs ? { disabled } : {}) });
         }
     }
     result.sort((a, b) => {