tinybase 4.0.0-beta.1 → 4.0.0-beta.2

package/lib/debug/persister-file.js

@@ -1,7 +1,6 @@
 import {readFile, writeFile} from 'fs/promises';
 import {watch} from 'fs';
 
-const EMPTY_STRING = '';
 const UTF8 = 'utf8';
 
 const arrayReduce = (array, cb, initial) => array.reduce(cb, initial);
@@ -40,43 +39,45 @@ const createCustomPersister = (
   let saves = 0;
   let listening = false;
   let listeningHandle;
+  const loadLock = async (actions) => {
+    /* istanbul ignore else */
+    if (loadSave != 2) {
+      loadSave = 1;
+      {
+        loads++;
+      }
+      await actions();
+      loadSave = 0;
+    }
+  };
   const persister = {
     load: async (initialTables, initialValues) => {
-      /* istanbul ignore else */
-      if (loadSave != 2) {
-        loadSave = 1;
-        {
-          loads++;
-        }
-        const body = await getPersisted();
-        if (!isUndefined(body) && body != EMPTY_STRING) {
-          store.setJson(body);
-        } else {
-          store.setContent([initialTables, initialValues]);
-        }
-        loadSave = 0;
-      }
+      await loadLock(async () => {
+        let content;
+        try {
+          content = await getPersisted();
+        } catch {}
+        store.setContent(content ?? [initialTables, initialValues]);
+      });
       return persister;
     },
     startAutoLoad: async (initialTables, initialValues) => {
       persister.stopAutoLoad();
       await persister.load(initialTables, initialValues);
       listening = true;
-      listeningHandle = addPersisterListener(async (content) => {
-        if (isUndefined(content)) {
-          await persister.load();
-        } else {
-          /* istanbul ignore else */
-          if (loadSave != 2) {
-            loadSave = 1;
-            {
-              loads++;
+      listeningHandle = addPersisterListener(
+        async (getContent, getTransactionChanges) => {
+          await loadLock(async () => {
+            if (getTransactionChanges) {
+              store.setTransactionChanges(getTransactionChanges());
+            } else {
+              try {
+                store.setContent(getContent?.() ?? (await getPersisted()));
+              } catch {}
             }
-            store.setContent(content);
-            loadSave = 0;
-          }
-        }
-      });
+          });
+        },
+      );
       return persister;
     },
     stopAutoLoad: () => {
@@ -87,21 +88,24 @@ const createCustomPersister = (
       }
       return persister;
     },
-    save: async () => {
+    save: async (getTransactionChanges) => {
       /* istanbul ignore else */
       if (loadSave != 1) {
         loadSave = 2;
         {
           saves++;
         }
-        await setPersisted(store.getContent);
+        await setPersisted(store.getContent, getTransactionChanges);
         loadSave = 0;
       }
       return persister;
     },
     startAutoSave: async () => {
       await persister.stopAutoSave().save();
-      listenerId = store.addDidFinishTransactionListener(persister.save);
+      listenerId = store.addDidFinishTransactionListener(
+        (_store, getTransactionChanges) =>
+          persister.save(getTransactionChanges),
+      );
       return persister;
     },
     stopAutoSave: () => {
@@ -116,11 +120,7 @@ const createCustomPersister = (
 };
 
 const createFilePersister = (store, filePath) => {
-  const getPersisted = async () => {
-    try {
-      return await readFile(filePath, UTF8);
-    } catch {}
-  };
+  const getPersisted = async () => JSON.parse(await readFile(filePath, UTF8));
   const setPersisted = async (getContent) => {
     try {
       await writeFile(filePath, jsonString(getContent()), UTF8);

package/lib/debug/persister-remote.js

@@ -1,5 +1,3 @@
-const EMPTY_STRING = '';
-
 const arrayReduce = (array, cb, initial) => array.reduce(cb, initial);
 
 const jsonString = (obj) =>
@@ -36,43 +34,45 @@ const createCustomPersister = (
   let saves = 0;
   let listening = false;
   let listeningHandle;
+  const loadLock = async (actions) => {
+    /* istanbul ignore else */
+    if (loadSave != 2) {
+      loadSave = 1;
+      {
+        loads++;
+      }
+      await actions();
+      loadSave = 0;
+    }
+  };
   const persister = {
     load: async (initialTables, initialValues) => {
-      /* istanbul ignore else */
-      if (loadSave != 2) {
-        loadSave = 1;
-        {
-          loads++;
-        }
-        const body = await getPersisted();
-        if (!isUndefined(body) && body != EMPTY_STRING) {
-          store.setJson(body);
-        } else {
-          store.setContent([initialTables, initialValues]);
-        }
-        loadSave = 0;
-      }
+      await loadLock(async () => {
+        let content;
+        try {
+          content = await getPersisted();
+        } catch {}
+        store.setContent(content ?? [initialTables, initialValues]);
+      });
       return persister;
     },
     startAutoLoad: async (initialTables, initialValues) => {
       persister.stopAutoLoad();
       await persister.load(initialTables, initialValues);
       listening = true;
-      listeningHandle = addPersisterListener(async (content) => {
-        if (isUndefined(content)) {
-          await persister.load();
-        } else {
-          /* istanbul ignore else */
-          if (loadSave != 2) {
-            loadSave = 1;
-            {
-              loads++;
+      listeningHandle = addPersisterListener(
+        async (getContent, getTransactionChanges) => {
+          await loadLock(async () => {
+            if (getTransactionChanges) {
+              store.setTransactionChanges(getTransactionChanges());
+            } else {
+              try {
+                store.setContent(getContent?.() ?? (await getPersisted()));
+              } catch {}
             }
-            store.setContent(content);
-            loadSave = 0;
-          }
-        }
-      });
+          });
+        },
+      );
       return persister;
     },
     stopAutoLoad: () => {
@@ -83,21 +83,24 @@ const createCustomPersister = (
       }
       return persister;
     },
-    save: async () => {
+    save: async (getTransactionChanges) => {
       /* istanbul ignore else */
       if (loadSave != 1) {
         loadSave = 2;
         {
           saves++;
         }
-        await setPersisted(store.getContent);
+        await setPersisted(store.getContent, getTransactionChanges);
         loadSave = 0;
       }
       return persister;
     },
     startAutoSave: async () => {
       await persister.stopAutoSave().save();
-      listenerId = store.addDidFinishTransactionListener(persister.save);
+      listenerId = store.addDidFinishTransactionListener(
+        (_store, getTransactionChanges) =>
+          persister.save(getTransactionChanges),
+      );
       return persister;
     },
     stopAutoSave: () => {
@@ -122,7 +125,7 @@ const createRemotePersister = (
   const getPersisted = async () => {
     const response = await fetch(loadUrl);
     lastEtag = getETag(response);
-    return response.text();
+    return JSON.parse(await response.text());
   };
   const setPersisted = async (getContent) =>
     await fetch(saveUrl, {

package/lib/debug/persister-yjs.js

@@ -1,4 +1,10 @@
-const EMPTY_STRING = '';
+import {Map} from 'yjs';
+
+const arrayForEach = (array, cb) => array.forEach(cb);
+const arrayMap = (array, cb) => array.map(cb);
+const arrayLength = (array) => array.length;
+const arrayIsEmpty = (array) => arrayLength(array) == 0;
+const arrayShift = (array) => array.shift();
 
 const isUndefined = (thing) => thing == void 0;
 const ifNotUndefined = (value, then, otherwise) =>
@@ -6,6 +12,17 @@ const ifNotUndefined = (value, then, otherwise) =>
 
 const object = Object;
 const objFreeze = object.freeze;
+const objNew = () => ({});
+const objGet = (obj, id) => ifNotUndefined(obj, (obj2) => obj2[id]);
+const objHas = (obj, id) => !isUndefined(objGet(obj, id));
+const objMap = (obj, cb) =>
+  arrayMap(object.entries(obj), ([id, value]) => cb(value, id));
+const objEnsure = (obj, id, getDefaultValue) => {
+  if (!objHas(obj, id)) {
+    obj[id] = getDefaultValue();
+  }
+  return obj[id];
+};
 
 const createCustomPersister = (
   store,
@@ -20,43 +37,45 @@ const createCustomPersister = (
   let saves = 0;
   let listening = false;
   let listeningHandle;
+  const loadLock = async (actions) => {
+    /* istanbul ignore else */
+    if (loadSave != 2) {
+      loadSave = 1;
+      {
+        loads++;
+      }
+      await actions();
+      loadSave = 0;
+    }
+  };
   const persister = {
     load: async (initialTables, initialValues) => {
-      /* istanbul ignore else */
-      if (loadSave != 2) {
-        loadSave = 1;
-        {
-          loads++;
-        }
-        const body = await getPersisted();
-        if (!isUndefined(body) && body != EMPTY_STRING) {
-          store.setJson(body);
-        } else {
-          store.setContent([initialTables, initialValues]);
-        }
-        loadSave = 0;
-      }
+      await loadLock(async () => {
+        let content;
+        try {
+          content = await getPersisted();
+        } catch {}
+        store.setContent(content ?? [initialTables, initialValues]);
+      });
       return persister;
     },
     startAutoLoad: async (initialTables, initialValues) => {
       persister.stopAutoLoad();
       await persister.load(initialTables, initialValues);
       listening = true;
-      listeningHandle = addPersisterListener(async (content) => {
-        if (isUndefined(content)) {
-          await persister.load();
-        } else {
-          /* istanbul ignore else */
-          if (loadSave != 2) {
-            loadSave = 1;
-            {
-              loads++;
+      listeningHandle = addPersisterListener(
+        async (getContent, getTransactionChanges) => {
+          await loadLock(async () => {
+            if (getTransactionChanges) {
+              store.setTransactionChanges(getTransactionChanges());
+            } else {
+              try {
+                store.setContent(getContent?.() ?? (await getPersisted()));
+              } catch {}
             }
-            store.setContent(content);
-            loadSave = 0;
-          }
-        }
-      });
+          });
+        },
+      );
       return persister;
     },
     stopAutoLoad: () => {
@@ -67,21 +86,24 @@ const createCustomPersister = (
       }
       return persister;
     },
-    save: async () => {
+    save: async (getTransactionChanges) => {
       /* istanbul ignore else */
       if (loadSave != 1) {
         loadSave = 2;
         {
           saves++;
         }
-        await setPersisted(store.getContent);
+        await setPersisted(store.getContent, getTransactionChanges);
         loadSave = 0;
       }
       return persister;
     },
     startAutoSave: async () => {
       await persister.stopAutoSave().save();
-      listenerId = store.addDidFinishTransactionListener(persister.save);
+      listenerId = store.addDidFinishTransactionListener(
+        (_store, getTransactionChanges) =>
+          persister.save(getTransactionChanges),
+      );
       return persister;
     },
     stopAutoSave: () => {
@@ -95,18 +117,181 @@ const createCustomPersister = (
   return objFreeze(persister);
 };
 
-const createYjsPersister = (store, yDoc) => {
-  const map = yDoc.getMap('tinybase/store');
-  const getPersisted = async () => map.get('json');
-  const setPersisted = async (getContent) => {
-    map.set('json', JSON.stringify(getContent()));
+const collForEach = (coll, cb) => coll?.forEach(cb);
+
+const mapForEach = (map, cb) =>
+  collForEach(map, (value, key) => cb(key, value));
+
+const tablesKey = 't';
+const valuesKey = 'v';
+const DELETE = 'delete';
+const ensureYContent = (yContent) => {
+  if (!yContent.size) {
+    yContent.set(tablesKey, new Map());
+    yContent.set(valuesKey, new Map());
+  }
+};
+const getYContent = (yContent) => [
+  yContent.get(tablesKey),
+  yContent.get(valuesKey),
+];
+const getTransactionChangesFromYDoc = (yContent, events) => {
+  if (arrayLength(events) == 1 && arrayIsEmpty(events[0].path)) {
+    return [yContent.get('t').toJSON(), yContent.get('v').toJSON()];
+  }
+  const [yTables, yValues] = getYContent(yContent);
+  const tables = {};
+  const values = {};
+  arrayForEach(events, ({path, changes: {keys}}) =>
+    arrayShift(path) == tablesKey
+      ? ifNotUndefined(
+          arrayShift(path),
+          (yTableId) => {
+            const table = objEnsure(tables, yTableId, objNew);
+            const yTable = yTables.get(yTableId);
+            ifNotUndefined(
+              arrayShift(path),
+              (yRowId) => {
+                const row = objEnsure(table, yRowId, objNew);
+                const yRow = yTable.get(yRowId);
+                mapForEach(
+                  keys,
+                  (cellId, {action}) =>
+                    (row[cellId] = action == DELETE ? null : yRow.get(cellId)),
+                );
+              },
+              () =>
+                mapForEach(
+                  keys,
+                  (rowId, {action}) =>
+                    (table[rowId] =
+                      action == DELETE ? null : yTable.get(rowId)?.toJSON()),
+                ),
+            );
+          },
+          () =>
+            mapForEach(
+              keys,
+              (tableId, {action}) =>
+                (tables[tableId] =
+                  action == DELETE ? null : yTables.get(tableId)?.toJSON()),
+            ),
+        )
+      : mapForEach(
+          keys,
+          (valueId, {action}) =>
+            (values[valueId] = action == DELETE ? null : yValues.get(valueId)),
+        ),
+  );
+  return [tables, values];
+};
+const setTransactionChangesToYDoc = (
+  yContent,
+  getContent,
+  getTransactionChanges,
+) => {
+  ensureYContent(yContent);
+  const [yTables, yValues] = getYContent(yContent);
+  const transactionChangesDidFail = () => {
+    transactionChangesFailed = 1;
   };
+  let transactionChangesFailed = 1;
+  ifNotUndefined(getTransactionChanges?.(), ([cellChanges, valueChanges]) => {
+    transactionChangesFailed = 0;
+    objMap(cellChanges, (table, tableId) =>
+      transactionChangesFailed
+        ? 0
+        : isUndefined(table)
+        ? yTables.delete(tableId)
+        : ifNotUndefined(
+            yTables.get(tableId),
+            (yTable) =>
+              objMap(table, (row, rowId) =>
+                transactionChangesFailed
+                  ? 0
+                  : isUndefined(row)
+                  ? yTable.delete(rowId)
+                  : ifNotUndefined(
+                      yTable.get(rowId),
+                      (yRow) =>
+                        objMap(row, (cell, cellId) =>
+                          isUndefined(cell)
+                            ? yRow.delete(cellId)
+                            : yRow.set(cellId, cell),
+                        ),
+                      transactionChangesDidFail,
+                    ),
+              ),
+            transactionChangesDidFail,
+          ),
+    );
+    objMap(valueChanges, (value, valueId) =>
+      transactionChangesFailed
+        ? 0
+        : isUndefined(value)
+        ? yValues.delete(valueId)
+        : yValues.set(valueId, value),
+    );
+  });
+  if (transactionChangesFailed) {
+    const [tables, values] = getContent();
+    yMapMatch(yTables, void 0, tables, (_, tableId, table) =>
+      yMapMatch(yTables, tableId, table, (yTable, rowId, row) =>
+        yMapMatch(yTable, rowId, row, (yRow, cellId, cell) => {
+          if (yRow.get(cellId) !== cell) {
+            yRow.set(cellId, cell);
+            return 1;
+          }
+        }),
+      ),
+    );
+    yMapMatch(yValues, void 0, values, (_, valueId, value) => {
+      if (yValues.get(valueId) !== value) {
+        yValues.set(valueId, value);
+      }
+    });
+  }
+};
+const yMapMatch = (yMapOrParent, idInParent, obj, set) => {
+  const yMap = isUndefined(idInParent)
+    ? yMapOrParent
+    : yMapOrParent.get(idInParent) ?? yMapOrParent.set(idInParent, new Map());
+  let changed;
+  objMap(obj, (value, id) => {
+    if (set(yMap, id, value)) {
+      changed = 1;
+    }
+  });
+  yMap.forEach((_, id) => {
+    if (!objHas(obj, id)) {
+      yMap.delete(id);
+      changed = 1;
+    }
+  });
+  if (!isUndefined(idInParent) && !yMap.size) {
+    yMapOrParent.delete(idInParent);
+  }
+  return changed;
+};
+const createYjsPersister = (store, yDoc, yMapName = 'tinybase') => {
+  const yContent = yDoc.getMap(yMapName);
+  const getPersisted = async () =>
+    yContent.size
+      ? [yContent.get('t').toJSON(), yContent.get('v').toJSON()]
+      : void 0;
+  const setPersisted = async (getContent, getTransactionChanges) =>
+    yDoc.transact(() =>
+      setTransactionChangesToYDoc(yContent, getContent, getTransactionChanges),
+    );
   const addPersisterListener = (listener) => {
-    const observer = () => listener();
-    map.observe(observer);
+    const observer = (events) =>
+      listener(void 0, () => getTransactionChangesFromYDoc(yContent, events));
+    yContent.observeDeep(observer);
     return observer;
   };
-  const delPersisterListener = (observer) => map.unobserve(observer);
+  const delPersisterListener = (observer) => {
+    yContent.unobserveDeep(observer);
+  };
   return createCustomPersister(
     store,
     getPersisted,

package/lib/debug/persisters.js

@@ -1,5 +1,3 @@
-const EMPTY_STRING = '';
-
 const isUndefined = (thing) => thing == void 0;
 const ifNotUndefined = (value, then, otherwise) =>
   isUndefined(value) ? otherwise?.() : then(value);
@@ -20,43 +18,45 @@ const createCustomPersister = (
   let saves = 0;
   let listening = false;
   let listeningHandle;
+  const loadLock = async (actions) => {
+    /* istanbul ignore else */
+    if (loadSave != 2) {
+      loadSave = 1;
+      {
+        loads++;
+      }
+      await actions();
+      loadSave = 0;
+    }
+  };
   const persister = {
     load: async (initialTables, initialValues) => {
-      /* istanbul ignore else */
-      if (loadSave != 2) {
-        loadSave = 1;
-        {
-          loads++;
-        }
-        const body = await getPersisted();
-        if (!isUndefined(body) && body != EMPTY_STRING) {
-          store.setJson(body);
-        } else {
-          store.setContent([initialTables, initialValues]);
-        }
-        loadSave = 0;
-      }
+      await loadLock(async () => {
+        let content;
+        try {
+          content = await getPersisted();
+        } catch {}
+        store.setContent(content ?? [initialTables, initialValues]);
+      });
       return persister;
     },
     startAutoLoad: async (initialTables, initialValues) => {
       persister.stopAutoLoad();
       await persister.load(initialTables, initialValues);
       listening = true;
-      listeningHandle = addPersisterListener(async (content) => {
-        if (isUndefined(content)) {
-          await persister.load();
-        } else {
-          /* istanbul ignore else */
-          if (loadSave != 2) {
-            loadSave = 1;
-            {
-              loads++;
+      listeningHandle = addPersisterListener(
+        async (getContent, getTransactionChanges) => {
+          await loadLock(async () => {
+            if (getTransactionChanges) {
+              store.setTransactionChanges(getTransactionChanges());
+            } else {
+              try {
+                store.setContent(getContent?.() ?? (await getPersisted()));
+              } catch {}
             }
-            store.setContent(content);
-            loadSave = 0;
-          }
-        }
-      });
+          });
+        },
+      );
       return persister;
     },
     stopAutoLoad: () => {
@@ -67,21 +67,24 @@ const createCustomPersister = (
       }
       return persister;
     },
-    save: async () => {
+    save: async (getTransactionChanges) => {
       /* istanbul ignore else */
       if (loadSave != 1) {
         loadSave = 2;
         {
           saves++;
         }
-        await setPersisted(store.getContent);
+        await setPersisted(store.getContent, getTransactionChanges);
         loadSave = 0;
       }
       return persister;
     },
     startAutoSave: async () => {
       await persister.stopAutoSave().save();
-      listenerId = store.addDidFinishTransactionListener(persister.save);
+      listenerId = store.addDidFinishTransactionListener(
+        (_store, getTransactionChanges) =>
+          persister.save(getTransactionChanges),
+      );
       return persister;
     },
     stopAutoSave: () => {

package/lib/debug/queries.js

@@ -44,6 +44,11 @@ const isFunction = (thing) => getTypeOf(thing) == FUNCTION;
 const isArray = (thing) => Array.isArray(thing);
 const getUndefined = () => void 0;
 
+const object = Object;
+const objFreeze = object.freeze;
+const objMap = (obj, cb) =>
+  arrayMap(object.entries(obj), ([id, value]) => cb(value, id));
+
 const collSize = (coll) => coll.size;
 const collHas = (coll, keyOrValue) => coll?.has(keyOrValue) ?? false;
 const collIsEmpty = (coll) => isUndefined(coll) || collSize(coll) == 0;
@@ -329,11 +334,6 @@ const getCreateFunction = (getFunction) => {
   };
 };
 
-const object = Object;
-const objFreeze = object.freeze;
-const objMap = (obj, cb) =>
-  arrayMap(object.entries(obj), ([id, value]) => cb(value, id));
-
 const createQueries = getCreateFunction((store) => {
   const createStore = store.createStore;
   const [