tinybase 8.6.0-beta.0 → 9.0.0-beta.1

package/persisters/persister-durable-object-sql-storage/index.js

@@ -51,6 +51,7 @@ const tryCatch = async (action, then1, then2) => {
   }
 };
 
+const arrayHas = (array, value) => array.includes(value);
 const arrayForEach = (array, cb) => array.forEach(cb);
 const arrayJoin = (array, sep = EMPTY_STRING) => array.join(sep);
 const arrayMap = (array, cb) => array.map(cb);
@@ -60,6 +61,14 @@ const arrayClear = (array, to) => array.splice(0, to);
 const arrayPush = (array, ...values) => array.push(...values);
 const arrayShift = (array) => array.shift();
 
+const collSize = (coll) => coll?.size ?? 0;
+const collHas = (coll, keyOrValue) => coll?.has(keyOrValue) ?? false;
+const collIsEmpty = (coll) => isUndefined(coll) || collSize(coll) == 0;
+const collValues = (coll) => [...(coll?.values() ?? [])];
+const collClear = (coll) => coll.clear();
+const collForEach = (coll, cb) => coll?.forEach(cb);
+const collDel = (coll, keyOrValue) => coll?.delete(keyOrValue);
+
 const object = Object;
 const getPrototypeOf = (obj) => object.getPrototypeOf(obj);
 const objEntries = object.entries;
@@ -115,13 +124,13 @@ const replaceUndefinedString = (obj) =>
         ? objMap(obj, replaceUndefinedString)
         : obj;
 
-const collSize = (coll) => coll?.size ?? 0;
-const collHas = (coll, keyOrValue) => coll?.has(keyOrValue) ?? false;
-const collIsEmpty = (coll) => isUndefined(coll) || collSize(coll) == 0;
-const collValues = (coll) => [...(coll?.values() ?? [])];
-const collClear = (coll) => coll.clear();
-const collForEach = (coll, cb) => coll?.forEach(cb);
-const collDel = (coll, keyOrValue) => coll?.delete(keyOrValue);
+const setNew = (entryOrEntries) =>
+  new Set(
+    isArray(entryOrEntries) || isUndefined(entryOrEntries)
+      ? entryOrEntries
+      : [entryOrEntries],
+  );
+const setAdd = (set, value) => set?.add(value);
 
 const map = Map;
 const mapNew = (entries) => new map(entries);
@@ -184,14 +193,6 @@ const getPoolFunctions = () => {
   ];
 };
 
-const setNew = (entryOrEntries) =>
-  new Set(
-    isArray(entryOrEntries) || isUndefined(entryOrEntries)
-      ? entryOrEntries
-      : [entryOrEntries],
-  );
-const setAdd = (set, value) => set?.add(value);
-
 const getWildcardedLeaves = (deepIdSet, path = [EMPTY_STRING]) => {
   const leaves = [];
   const deep = (node, p) =>
@@ -364,13 +365,14 @@ const createCustomPersister = (
             const content = await getPersisted();
             if (isArray(content)) {
               setContentOrChanges(content);
-            } else if (initialContent) {
+            } else if (isUndefined(content) && initialContent) {
               setDefaultContent(initialContent);
-            } else {
+            } else if (!isUndefined(content)) {
               errorNew(`Content is not an array: ${content}`);
             }
           },
-          () => {
+          (error) => {
+            onIgnoredError?.(error);
             if (initialContent) {
               setDefaultContent(initialContent);
             }
@@ -573,6 +575,8 @@ const DEFAULT_TABULAR_VALUES_CONFIG = {
   save: 0,
   [TABLE_NAME]: TINYBASE + '_values',
 };
+const getValuesIn = (config) => (isArray(config) ? config : void 0);
+const hasValues = (config) => isTrue(config) || isArray(config);
 const getDefaultedConfig = (configOrStoreTableName) =>
   objMerge(
     DEFAULT_CONFIG,
@@ -623,11 +627,15 @@ const getConfigStructures = (configOrStoreTableName) => {
     ];
   }
   const {tables: {load = {}, save = {}} = {}, values = {}} = config;
-  const valuesConfig = slice(
-    objValues(objMerge(DEFAULT_TABULAR_VALUES_CONFIG, values)),
-    0,
-    objSize(DEFAULT_TABULAR_VALUES_CONFIG),
-  );
+  const valuesConfigObj = objMerge(DEFAULT_TABULAR_VALUES_CONFIG, values);
+  const {load: valuesLoad, save: valuesSave} = valuesConfigObj;
+  const valuesConfig = [
+    hasValues(valuesLoad),
+    hasValues(valuesSave),
+    valuesConfigObj[TABLE_NAME],
+    getValuesIn(valuesLoad),
+    getValuesIn(valuesSave),
+  ];
   const valuesTable = valuesConfig[2];
   const managedTableNames = setNew(valuesTable);
   const excludedTableNames = setNew(valuesTable);
@@ -689,8 +697,18 @@ const getCommandFunctions = (
       ({tn, cn}) => setAdd(mapEnsure(schemaMap, tn, setNew), cn),
     );
   };
-  const loadTable = async (tableName, rowIdColumnName, condition) =>
-    canSelect(tableName, rowIdColumnName)
+  const loadTable = async (
+    tableName,
+    rowIdColumnName,
+    condition,
+    contentSubIds,
+  ) => {
+    const contentSubIdSet = isUndefined(contentSubIds)
+      ? void 0
+      : setNew(contentSubIds);
+    const includeContentSubId = (contentSubId) =>
+      isUndefined(contentSubIdSet) || collHas(contentSubIdSet, contentSubId);
+    return canSelect(tableName, rowIdColumnName)
       ? objNew(
           arrayFilter(
             arrayMap(
@@ -699,17 +717,35 @@ const getCommandFunctions = (
                   escapeId(tableName) +
                   getWhereCondition(tableName, condition),
               ),
-              (row) => [
-                row[rowIdColumnName],
-                decode
+              (row) => {
+                const rowId = row[rowIdColumnName];
+                const rowContent = decode
                   ? objMap(objDel(row, rowIdColumnName), decode)
-                  : objDel(row, rowIdColumnName),
-              ],
+                  : objDel(row, rowIdColumnName);
+                return [
+                  rowId,
+                  isUndefined(contentSubIdSet)
+                    ? rowContent
+                    : objNew(
+                        arrayFilter(
+                          objToArray(
+                            rowContent,
+                            (cellOrValue, contentSubId) => [
+                              contentSubId,
+                              cellOrValue,
+                            ],
+                          ),
+                          ([contentSubId]) => includeContentSubId(contentSubId),
+                        ),
+                      ),
+                ];
+              },
             ),
             (entry) => !isUndefined(entry[0]) && !objIsEmpty(entry[1]),
           ),
         )
       : {};
+  };
   const saveTable = async (
     tableName,
     rowIdColumnName,
@@ -718,15 +754,23 @@ const getCommandFunctions = (
     deleteEmptyTable,
     partial = false,
     condition = TRUE,
+    contentSubIds,
   ) => {
+    const contentSubIdSet = isUndefined(contentSubIds)
+      ? void 0
+      : setNew(contentSubIds);
+    const includeContentSubId = (contentSubId) =>
+      isUndefined(contentSubIdSet) || collHas(contentSubIdSet, contentSubId);
     const settingColumnNameSet = setNew();
     objMap(content ?? {}, (contentRow) =>
-      arrayMap(objIds(contentRow ?? {}), (cellOrValueId) =>
-        setAdd(settingColumnNameSet, cellOrValueId),
+      arrayMap(
+        arrayFilter(objIds(contentRow ?? {}), includeContentSubId),
+        (cellOrValueId) => setAdd(settingColumnNameSet, cellOrValueId),
       ),
     );
     const settingColumnNames = collValues(settingColumnNameSet);
     if (
+      isUndefined(contentSubIdSet) &&
       !partial &&
       deleteEmptyTable &&
       condition == TRUE &&
@@ -738,7 +782,13 @@ const getCommandFunctions = (
       return;
     }
     const currentColumnNames = mapGet(schemaMap, tableName);
-    const unaccountedColumnNames = setNew(collValues(currentColumnNames));
+    const unaccountedColumnNames = setNew(
+      arrayFilter(
+        collValues(currentColumnNames),
+        (columnName) =>
+          columnName == rowIdColumnName || includeContentSubId(columnName),
+      ),
+    );
     if (!arrayIsEmpty(settingColumnNames)) {
       if (!collHas(schemaMap, tableName)) {
         await databaseExecuteCommand(
@@ -804,11 +854,13 @@ const getCommandFunctions = (
     ]);
     if (partial) {
       if (isUndefined(content)) {
-        await databaseExecuteCommand(
-          DELETE_FROM +
-            escapeId(tableName) +
-            getWhereCondition(tableName, condition),
-        );
+        if (isUndefined(contentSubIdSet)) {
+          await databaseExecuteCommand(
+            DELETE_FROM +
+              escapeId(tableName) +
+              getWhereCondition(tableName, condition),
+          );
+        }
       } else {
         await promiseAll(
           objToArray(content, async (row, rowId) => {
@@ -821,18 +873,24 @@ const getCommandFunctions = (
                 [rowId],
               );
             } else if (!arrayIsEmpty(settingColumnNames)) {
-              await upsert(
-                databaseExecuteCommand,
-                tableName,
-                rowIdColumnName,
+              const changingColumnNames = arrayFilter(
                 objIds(row),
-                {
-                  [rowId]: encode
-                    ? arrayMap(objValues(row), encode)
-                    : objValues(row),
-                },
-                currentColumnNames,
+                includeContentSubId,
               );
+              if (!arrayIsEmpty(changingColumnNames)) {
+                await upsert(
+                  databaseExecuteCommand,
+                  tableName,
+                  rowIdColumnName,
+                  changingColumnNames,
+                  {
+                    [rowId]: arrayMap(changingColumnNames, (cellId) =>
+                      encode ? encode(row[cellId]) : row[cellId],
+                    ),
+                  },
+                  currentColumnNames,
+                );
+              }
             }
           }),
         );
@@ -841,7 +899,9 @@ const getCommandFunctions = (
       if (!arrayIsEmpty(settingColumnNames)) {
         const changingColumnNames = arrayFilter(
           collValues(mapGet(schemaMap, tableName)),
-          (changingColumnName) => changingColumnName != rowIdColumnName,
+          (changingColumnName) =>
+            changingColumnName != rowIdColumnName &&
+            includeContentSubId(changingColumnName),
         );
         const rows = {};
         const deleteRowIds = [];
@@ -857,6 +917,7 @@ const getCommandFunctions = (
           rowIdColumnName,
           changingColumnNames,
           rows,
+          currentColumnNames,
         );
         await databaseExecuteCommand(
           DELETE_FROM +
@@ -865,7 +926,10 @@ const getCommandFunctions = (
             `AND${escapeId(rowIdColumnName)}NOT IN(${getPlaceholders(deleteRowIds)})`,
           deleteRowIds,
         );
-      } else if (collHas(schemaMap, tableName)) {
+      } else if (
+        isUndefined(contentSubIdSet) &&
+        collHas(schemaMap, tableName)
+      ) {
         await databaseExecuteCommand(
           DELETE_FROM +
             escapeId(tableName) +
@@ -1004,7 +1068,13 @@ const createTabularPersister = (
   [
     tablesLoadConfig,
     tablesSaveConfig,
-    [valuesLoad, valuesSave, valuesTableName],
+    [
+      valuesLoad,
+      valuesSave,
+      valuesTableName,
+      valuesLoadValueIds,
+      valuesSaveValueIds,
+    ],
   ],
   managedTableNames,
   querySchema,
@@ -1054,15 +1124,33 @@ const createTabularPersister = (
         },
       ),
     );
+  const getValuesSubset = (values, valueIds, partial) =>
+    objNew(
+      partial == true
+        ? arrayFilter(
+            objToArray(values, (value, valueId) => [valueId, value]),
+            ([valueId]) => arrayHas(valueIds, valueId),
+          )
+        : arrayFilter(
+            arrayMap(valueIds, (valueId) => [valueId, values[valueId]]),
+            ([, value]) => !isUndefined(value),
+          ),
+    );
   const saveValues = async (values, partial) =>
     valuesSave
       ? await saveTable(
           valuesTableName,
           DEFAULT_ROW_ID_COLUMN_NAME,
-          {[SINGLE_ROW_ID]: values},
+          {
+            [SINGLE_ROW_ID]: isUndefined(valuesSaveValueIds)
+              ? values
+              : getValuesSubset(values, valuesSaveValueIds, partial),
+          },
           true,
           true,
           partial,
+          void 0,
+          valuesSaveValueIds,
         )
       : null;
   const loadTables = async () =>
@@ -1082,9 +1170,14 @@ const createTabularPersister = (
     );
   const loadValues = async () =>
     valuesLoad
-      ? (await loadTable(valuesTableName, DEFAULT_ROW_ID_COLUMN_NAME))[
-          SINGLE_ROW_ID
-        ]
+      ? (
+          await loadTable(
+            valuesTableName,
+            DEFAULT_ROW_ID_COLUMN_NAME,
+            void 0,
+            valuesLoadValueIds,
+          )
+        )[SINGLE_ROW_ID]
       : {};
   const getPersisted = () =>
     transaction(async () => {
@@ -1217,6 +1310,8 @@ const createCustomSqlitePersister = (
   );
 };
 
+const getSqlRows = (sqlStorage, sql, ...params) =>
+  sqlStorage.exec(sql, ...params).toArray();
 const createDurableObjectSqlStoragePersister = (
   store,
   sqlStorage,
@@ -1241,7 +1336,7 @@ const createDurableObjectSqlStoragePersister = (
     async (sql, params = []) => {
       if (!['BEGIN', 'END'].includes(sql)) {
         sql = sql.replace(/\$\d+/g, '?');
-        return sqlStorage.exec(sql, ...params).toArray();
+        return getSqlRows(sqlStorage, sql, ...params);
       }
       return [];
     },
@@ -1266,6 +1361,10 @@ const createDurableObjectFragmentedSqlStoragePersister = (
   const tablePrefix = storagePrefix.replace(/[^a-zA-Z0-9_]/g, '_');
   const tablesTable = `${tablePrefix}tinybase_tables`;
   const valuesTable = `${tablePrefix}tinybase_values`;
+  const insertTableSql = `INSERT INTO ${tablesTable} (type, table_id, row_id, cell_id, value_data, timestamp, hash) VALUES (?, ?, ?, ?, ?, ?, ?)`;
+  const insertValueSql = `INSERT INTO ${valuesTable} (value_id, value_data, timestamp, hash) VALUES (?, ?, ?, ?)`;
+  const getRowKey = (tableId, rowId) =>
+    jsonStringWithUndefined([tableId, rowId]);
   const initializeTables = () => {
     sqlStorage.exec(`
       CREATE TABLE IF NOT EXISTS ${tablesTable} (
@@ -1288,11 +1387,74 @@ const createDurableObjectFragmentedSqlStoragePersister = (
     `);
   };
   initializeTables();
+  const getCondition = (params, column, value) =>
+    isNull(value)
+      ? `${column} IS NULL`
+      : (arrayPush(params, value), `${column} = ?`);
+  const delTablesRow = (tableId, rowId, cellId) => {
+    const params = [T];
+    sqlStorage.exec(
+      `DELETE FROM ${tablesTable} WHERE type = ? AND ` +
+        getCondition(params, 'table_id', tableId) +
+        ` AND ` +
+        getCondition(params, 'row_id', rowId) +
+        ` AND ` +
+        getCondition(params, 'cell_id', cellId),
+      ...params,
+    );
+  };
+  const setTablesRow = (tableId, rowId, cellId, valueData, timestamp, hash) => {
+    delTablesRow(tableId, rowId, cellId);
+    sqlStorage.exec(
+      insertTableSql,
+      T,
+      tableId,
+      rowId,
+      cellId,
+      valueData,
+      timestamp,
+      hash,
+    );
+  };
+  const delCellRows = (tableId, rowId) =>
+    sqlStorage.exec(
+      `DELETE FROM ${tablesTable} WHERE type = ? AND table_id = ? AND row_id = ? AND cell_id IS NOT NULL`,
+      T,
+      tableId,
+      rowId,
+    );
+  const delValuesRow = (valueId) => {
+    const params = [];
+    sqlStorage.exec(
+      `DELETE FROM ${valuesTable} WHERE ` +
+        getCondition(params, 'value_id', valueId),
+      ...params,
+    );
+  };
+  const setValuesRow = (valueId, valueData, timestamp, hash) => {
+    delValuesRow(valueId);
+    sqlStorage.exec(insertValueSql, valueId, valueData, timestamp, hash);
+  };
   const getPersisted = async () => {
     const tables = stampNewObjectWithHash();
     const values = stampNewObjectWithHash();
-    const tablesResult = sqlStorage.exec(`SELECT * FROM ${tablesTable}`);
-    for (const row of tablesResult.toArray()) {
+    const rowDataKeys = setNew();
+    const tablesRows = getSqlRows(sqlStorage, `SELECT * FROM ${tablesTable}`);
+    arrayForEach(tablesRows, (row) => {
+      const table_id = row.table_id ? string(row.table_id) : null;
+      const row_id = row.row_id ? string(row.row_id) : null;
+      const cell_id = row.cell_id ? string(row.cell_id) : null;
+      const value_data = string(row.value_data);
+      const timestamp = string(row.timestamp);
+      const hash = number(row.hash);
+      const [zeroOrCells] = jsonParseWithUndefined(value_data);
+      if (table_id && row_id && !cell_id && isObject(zeroOrCells)) {
+        const table = objEnsure(tables[0], table_id, stampNewObjectWithHash);
+        table[0][row_id] = [zeroOrCells, timestamp, hash];
+        setAdd(rowDataKeys, getRowKey(table_id, row_id));
+      }
+    });
+    arrayForEach(tablesRows, (row) => {
       const type = string(row.type);
       const table_id = row.table_id ? string(row.table_id) : null;
       const row_id = row.row_id ? string(row.row_id) : null;
@@ -1300,16 +1462,36 @@ const createDurableObjectFragmentedSqlStoragePersister = (
       const value_data = string(row.value_data);
       const timestamp = string(row.timestamp);
       const hash = number(row.hash);
-      const [zeroOrCellOrValue] = JSON.parse(value_data);
+      const [zeroOrCellOrValue] = jsonParseWithUndefined(value_data);
       if (type === T) {
         if (table_id && row_id && cell_id) {
-          const table = objEnsure(tables[0], table_id, stampNewObjectWithHash);
-          const tableRow = objEnsure(table[0], row_id, stampNewObjectWithHash);
-          tableRow[0][cell_id] = [zeroOrCellOrValue, timestamp, hash];
+          if (!collHas(rowDataKeys, getRowKey(table_id, row_id))) {
+            const table = objEnsure(
+              tables[0],
+              table_id,
+              stampNewObjectWithHash,
+            );
+            const tableRow = objEnsure(
+              table[0],
+              row_id,
+              stampNewObjectWithHash,
+            );
+            tableRow[0][cell_id] = [zeroOrCellOrValue, timestamp, hash];
+          }
         } else if (table_id && row_id) {
-          const table = objEnsure(tables[0], table_id, stampNewObjectWithHash);
-          const tableRow = objEnsure(table[0], row_id, stampNewObjectWithHash);
-          stampUpdate(tableRow, timestamp, hash);
+          if (!isObject(zeroOrCellOrValue)) {
+            const table = objEnsure(
+              tables[0],
+              table_id,
+              stampNewObjectWithHash,
+            );
+            const tableRow = objEnsure(
+              table[0],
+              row_id,
+              stampNewObjectWithHash,
+            );
+            stampUpdate(tableRow, timestamp, hash);
+          }
         } else if (table_id) {
           const table = objEnsure(tables[0], table_id, stampNewObjectWithHash);
           stampUpdate(table, timestamp, hash);
@@ -1317,20 +1499,22 @@ const createDurableObjectFragmentedSqlStoragePersister = (
           stampUpdate(tables, timestamp, hash);
         }
       }
-    }
-    const valuesResult = sqlStorage.exec(`SELECT * FROM ${valuesTable}`);
-    for (const row of valuesResult.toArray()) {
-      const value_id = row.value_id ? string(row.value_id) : null;
-      const value_data = string(row.value_data);
-      const timestamp = string(row.timestamp);
-      const hash = number(row.hash);
-      const [zeroOrCellOrValue] = JSON.parse(value_data);
-      if (value_id) {
-        values[0][value_id] = [zeroOrCellOrValue, timestamp, hash];
-      } else {
-        stampUpdate(values, timestamp, hash);
-      }
-    }
+    });
+    arrayForEach(
+      getSqlRows(sqlStorage, `SELECT * FROM ${valuesTable}`),
+      (row) => {
+        const value_id = row.value_id ? string(row.value_id) : null;
+        const value_data = string(row.value_data);
+        const timestamp = string(row.timestamp);
+        const hash = number(row.hash);
+        const [zeroOrCellOrValue] = jsonParseWithUndefined(value_data);
+        if (value_id) {
+          values[0][value_id] = [zeroOrCellOrValue, timestamp, hash];
+        } else {
+          stampUpdate(values, timestamp, hash);
+        }
+      },
+    );
     return [tables, values];
   };
   const setPersisted = async (
@@ -1340,64 +1524,42 @@ const createDurableObjectFragmentedSqlStoragePersister = (
       [valuesObj, valuesTime, valuesHash],
     ] = getContent(),
   ) => {
-    sqlStorage.exec(
-      `INSERT OR REPLACE INTO ${tablesTable} (type, table_id, row_id, cell_id, value_data, timestamp, hash) VALUES (?, ?, ?, ?, ?, ?, ?)`,
-      T,
+    const [fullTablesObj] = getContent()[0];
+    setTablesRow(
       null,
       null,
       null,
-      JSON.stringify([0]),
+      jsonStringWithUndefined([0]),
       tablesTime,
       tablesHash,
     );
     objForEach(tablesObj, ([tableObj, tableTime, tableHash], tableId) => {
-      sqlStorage.exec(
-        `INSERT OR REPLACE INTO ${tablesTable} (type, table_id, row_id, cell_id, value_data, timestamp, hash) VALUES (?, ?, ?, ?, ?, ?, ?)`,
-        T,
+      setTablesRow(
         tableId,
         null,
         null,
-        JSON.stringify([0]),
+        jsonStringWithUndefined([0]),
         tableTime,
         tableHash,
       );
       objForEach(tableObj, ([rowObj, rowTime, rowHash], rowId) => {
-        sqlStorage.exec(
-          `INSERT OR REPLACE INTO ${tablesTable} (type, table_id, row_id, cell_id, value_data, timestamp, hash) VALUES (?, ?, ?, ?, ?, ?, ?)`,
-          T,
+        const fullRowObj = fullTablesObj[tableId]?.[0]?.[rowId]?.[0] ?? rowObj;
+        delCellRows(tableId, rowId);
+        setTablesRow(
           tableId,
           rowId,
           null,
-          JSON.stringify([0]),
+          jsonStringWithUndefined([fullRowObj]),
           rowTime,
           rowHash,
         );
-        objForEach(rowObj, (cellStamp, cellId) => {
-          sqlStorage.exec(
-            `INSERT OR REPLACE INTO ${tablesTable} (type, table_id, row_id, cell_id, value_data, timestamp, hash) VALUES (?, ?, ?, ?, ?, ?, ?)`,
-            T,
-            tableId,
-            rowId,
-            cellId,
-            JSON.stringify([cellStamp[0]]),
-            cellStamp[1],
-            cellStamp[2],
-          );
-        });
       });
     });
-    sqlStorage.exec(
-      `INSERT OR REPLACE INTO ${valuesTable} (value_id, value_data, timestamp, hash) VALUES (?, ?, ?, ?)`,
-      null,
-      JSON.stringify([0]),
-      valuesTime,
-      valuesHash,
-    );
+    setValuesRow(null, jsonStringWithUndefined([0]), valuesTime, valuesHash);
     objForEach(valuesObj, (valueStamp, valueId) => {
-      sqlStorage.exec(
-        `INSERT OR REPLACE INTO ${valuesTable} (value_id, value_data, timestamp, hash) VALUES (?, ?, ?, ?)`,
+      setValuesRow(
         valueId,
-        JSON.stringify([valueStamp[0]]),
+        jsonStringWithUndefined([valueStamp[0]]),
         valueStamp[1],
         valueStamp[2],
       );