react-native-mosquito-transport 0.0.21 → 0.0.23

package/src/products/database/accessor.js

@@ -2,16 +2,20 @@ import { niceHash, shuffleArray, sortArrayByObjectKey } from "../../helpers/peri
 import { awaitStore, updateCacheStore } from "../../helpers/utils";
 import { CacheStore, Scoped } from "../../helpers/variables";
 import { assignExtractionFind, CompareBson, confirmFilterDoc, defaultBSON, downcastBSON, validateCollectionName, validateFilter } from "./validator";
-import getLodash from 'lodash.get';
-import setLodash from 'lodash.set';
-import unsetLodash from 'lodash.unset';
+import getLodash from 'lodash/get';
+import setLodash from 'lodash/set';
+import unsetLodash from 'lodash/unset';
 import { DatabaseRecordsListener } from "../../helpers/listeners";
-import cloneDeep from "lodash.clonedeep";
+import cloneDeep from "lodash/cloneDeep";
 import { BSONRegExp, ObjectId, Timestamp } from "bson";
 import { niceGuard, Validator } from "guard-object";
 import { TIMESTAMP } from "../..";
-import { decrementDatabaseSize, incrementDatabaseSize } from "./counter";
-import { serializeToBase64 } from "./bson";
+import { docSize, incrementDatabaseSize } from "./counter";
+import { deserializeBSON, serializeToBase64 } from "./bson";
+import { openDB, SQLITE_COMMANDS, SQLITE_PATH, useSqliteLinearAccessId } from "../../helpers/sqlite_manager";
+import { getStoreID, handleBigData, parseBigData } from "../../helpers/fs_manager";
+
+const { LIMITER_DATA, LIMITER_RESULT, DB_COUNT_QUERY } = SQLITE_PATH;
 
 export const listenQueryEntry = (callback, { accessId, builder, config, processId }) => {
     const { projectUrl, dbName, dbUrl, path } = builder;
@@ -25,8 +29,8 @@ export const listenQueryEntry = (callback, { accessId, builder, config, processI
 
     const listener = DatabaseRecordsListener.listenTo('d', async (dispatchId) => {
         if (dispatchId !== processId) return;
-        const cache = await getRecord(builder, config, accessId);
-        if (cache) callback(cache[episode]);
+        const cache = await getRecord(builder, accessId, episode);
+        if (cache) callback(cache[0]);
     });
 
     return () => {
@@ -40,100 +44,248 @@ export const listenQueryEntry = (callback, { accessId, builder, config, processI
     };
 };
 
-export const insertRecord = async (builder, config, accessId, value) => {
+export const insertCountQuery = async (builder, access_id, value) => {
+    const { projectUrl, dbUrl, dbName, path } = builder;
+
+    const { io } = Scoped.ReleaseCacheData;
+    if (io) {
+        setLodash(CacheStore.DatabaseCountResult, [projectUrl, dbUrl, dbName, path, access_id], { value, touched: Date.now() });
+    } else {
+        const initNode = `${projectUrl}_${dbUrl}_${dbName}_${path}`;
+        await useSqliteLinearAccessId(builder, access_id, 'dbQueryCount')(async sqlite => {
+            if (!Scoped.initedSqliteInstances.dbQueryCount[initNode]) {
+                Scoped.initedSqliteInstances.dbQueryCount[initNode] = (async () => {
+                    await sqlite.executeSql(`CREATE TABLE IF NOT EXISTS ${DB_COUNT_QUERY(path)} ( access_id TEXT PRIMARY KEY, value TEXT, touched INTEGER )`).catch(() => null);
+                    await Promise.allSettled([
+                        sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX(DB_COUNT_QUERY(path), ['access_id'])),
+                        // sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX(DB_COUNT_QUERY(path), ['touched']))
+                    ]);
+                })();
+            }
+
+            await Scoped.initedSqliteInstances.dbQueryCount[initNode];
+            await sqlite.executeSql(
+                SQLITE_COMMANDS.MERGE(DB_COUNT_QUERY(path), ['access_id', 'value', 'touched']),
+                [access_id, JSON.stringify(value), Date.now()]
+            );
+            setLodash(CacheStore.DatabaseStats.counters, [projectUrl, dbUrl, dbName, path], true);
+        });
+    }
+    updateCacheStore(undefined, ['DatabaseCountResult'])
+}
+
+export const getCountQuery = async (builder, access_id) => {
+    const { projectUrl, dbUrl, dbName, path } = builder;
+    const { io } = Scoped.ReleaseCacheData;
+
+    if (io) {
+        const data = getLodash(CacheStore.DatabaseCountResult, [projectUrl, dbUrl, dbName, path, access_id]);
+        if (data) data.touched = Date.now();
+        return data && data.value;
+    } else {
+        const result = await useSqliteLinearAccessId(builder, access_id, 'dbQueryCount')(sqlite =>
+            sqlite.executeSql(`SELECT * FROM ${DB_COUNT_QUERY(path)} WHERE access_id = ?`, [access_id]).then(async r => {
+                r = JSON.parse(r[0].rows.item(0).value);
+                await sqlite.executeSql(SQLITE_COMMANDS.UPDATE_COLUMNS(DB_COUNT_QUERY(path), ['touched'], 'access_id = ?'), [Date.now(), access_id]);
+                return r;
+            }).catch(() => undefined)
+        );
+        return result;
+    }
+}
+
+export const insertRecord = async (builder, config, accessIdWithoutLimit, value, episode = 0) => {
     builder = builder && cloneDeep(builder);
     config = config && cloneDeep(config);
     value = value && cloneDeep(value);
 
     await awaitStore();
+    const { io } = Scoped.ReleaseCacheData;
     const { projectUrl, dbUrl, dbName, path, command } = builder;
-    const entityId = await generateRecordID({}, config);
-    const colData = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'data', entityId], { config, command, listing: [] });
-    const trackedData = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'record', accessId]);
+    const { limit } = command;
+    const thisSize = docSize(value);
+
+    if (!io) {
+        await useSqliteLinearAccessId(builder, accessIdWithoutLimit, 'database')(async (sqlite, db_filename) => {
+            const initNode = `${projectUrl}_${dbUrl}_${dbName}_${path}`;
+
+            if (!Scoped.initedSqliteInstances.database[initNode]) {
+                Scoped.initedSqliteInstances.database[initNode] = (async () => {
+                    await Promise.allSettled([
+                        sqlite.executeSql(`CREATE TABLE IF NOT EXISTS ${LIMITER_DATA(path)} ( access_id TEXT PRIMARY KEY, value BLOB, touched INTEGER, size INTEGER )`),
+                        sqlite.executeSql(`CREATE TABLE IF NOT EXISTS ${LIMITER_RESULT(path)} ( access_id_limiter TEXT PRIMARY KEY, access_id TEXT, value BLOB, touched INTEGER, size INTEGER )`)
+                    ]);
+
+                    await Promise.allSettled([
+                        sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX(LIMITER_DATA(path), ['access_id'])),
+                        // sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX(LIMITER_DATA(path), ['touched'])),
+                        sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX(LIMITER_RESULT(path), ['access_id_limiter'])),
+                        // sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX(LIMITER_RESULT(path), ['touched']))
+                    ]);
+                })();
+            }
 
-    const newList = value ? Array.isArray(value) ? value : [value] : [];
+            await Scoped.initedSqliteInstances.database[initNode];
 
-    const { tracks, ignore, registeredOn } = trackedData || {};
-    const trackedList = [...tracks || []];
-    const ignoreList = [...ignore || []];
+            const resultAccessId = `${accessIdWithoutLimit}-${limit}`;
 
-    const addSet = (arr, _id) => {
-        const dex = arr.findIndex(v => CompareBson.equal(v, _id));
-        if (dex === -1) arr.push(_id);
-    }
+            const [instanceData, resultData] = await Promise.all([
+                sqlite.executeSql(`SELECT access_id, size FROM ${LIMITER_DATA(path)} WHERE access_id = ?`, [accessIdWithoutLimit]),
+                sqlite.executeSql(`SELECT access_id_limiter, size FROM ${LIMITER_RESULT(path)} WHERE access_id_limiter = ?`, [resultAccessId])
+            ]).then(r =>
+                r.map(v => v[0].rows.item(0))
+            );
+            const isEpisode = episode === 1 || !!resultData;
 
-    const deleteSet = (arr, _id) => {
-        const dex = arr.findIndex(v => CompareBson.equal(v, _id));
-        if (dex !== -1) arr.splice(dex, 1);
+            const editionSizeOffset = thisSize - (instanceData?.size || 0);
+            const resultSizeOffset = isEpisode ? thisSize - (resultData?.size || 0) : 0;
+
+            const newData = serializeToBase64({
+                command,
+                config,
+                latest_limiter: limit,
+                size: thisSize,
+                data: value ? Array.isArray(value) ? value : [value] : []
+            });
+            const newResultData = isEpisode && serializeToBase64({
+                data: value,
+                size: thisSize
+            });
+            const [blobData, episodeBlobData] = await Promise.all([
+                handleBigData(getStoreID(db_filename, LIMITER_DATA(path), accessIdWithoutLimit), newData),
+                isEpisode ?
+                    handleBigData(getStoreID(db_filename, LIMITER_RESULT(path), resultAccessId), newResultData)
+                    : Promise.resolve()
+            ]);
+
+            await Promise.all([
+                sqlite.executeSql(
+                    SQLITE_COMMANDS.MERGE(LIMITER_DATA(path), ['access_id', 'value', 'touched', 'size']),
+                    [accessIdWithoutLimit, blobData, Date.now(), thisSize]
+                ),
+                isEpisode ?
+                    sqlite.executeSql(
+                        SQLITE_COMMANDS.MERGE(LIMITER_RESULT(path), ['access_id_limiter', 'access_id', 'value', 'touched', 'size']),
+                        [resultAccessId, accessIdWithoutLimit, episodeBlobData, Date.now(), thisSize]
+                    ) : Promise.resolve()
+            ]);
+            incrementDatabaseSize(builder, path, editionSizeOffset + resultSizeOffset);
+        });
+        updateCacheStore(undefined, ['DatabaseStore', 'DatabaseStats']);
+        return;
     }
 
-    newList.forEach(e => {
-        const b4DocIndex = colData.listing.findIndex(v => CompareBson.equal(v._id, e._id));
-        if (b4DocIndex === -1) {
-            colData.listing.push(e);
-            incrementDatabaseSize(projectUrl, e);
-        } else {
-            decrementDatabaseSize(projectUrl, colData.listing[b4DocIndex]);
-            incrementDatabaseSize(projectUrl, e);
-            colData.listing[b4DocIndex] = e;
-        }
-        addSet(trackedList, e._id);
-    });
+    const instanceData = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'instance', accessIdWithoutLimit]);
+    const resultData = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'episode', accessIdWithoutLimit, limit]);
+    const isEpisode = episode === 1 || !!resultData;
 
-    (tracks || []).forEach(e => {
-        if (newList.findIndex(v => CompareBson.equal(v._id, e)) === -1) {
-            if (colData.listing.findIndex(v => CompareBson.equal(v._id, e)) === -1) {
-                deleteSet(trackedList, e);
-                deleteSet(ignoreList, e);
-            } else addSet(ignoreList, e);
-        } else deleteSet(ignoreList, e);
-    });
+    const editionSizeOffset = thisSize - (instanceData?.size || 0);
+    const resultSizeOffset = isEpisode ? thisSize - (resultData?.size || 0) : 0;
 
-    setLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'data', entityId], colData);
-    setLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'record', accessId], {
+    const newData = {
         command,
-        result: value,
-        tracks: [...trackedList],
-        ignore: [...ignoreList],
-        registeredOn: registeredOn || Date.now(),
-        updatedOn: Date.now()
-    });
-    updateCacheStore();
+        config,
+        latest_limiter: limit,
+        size: thisSize,
+        data: value ? Array.isArray(value) ? value : [value] : [],
+        touched: Date.now()
+    };
+    const newResultData = isEpisode && {
+        data: value,
+        size: thisSize,
+        touched: Date.now()
+    };
+
+    incrementDatabaseSize(builder, path, editionSizeOffset + resultSizeOffset);
+
+    setLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'instance', accessIdWithoutLimit], newData);
+    if (isEpisode) setLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'episode', accessIdWithoutLimit, limit], cloneDeep(newResultData));
+    updateCacheStore(undefined, ['DatabaseStore', 'DatabaseStats']);
 };
 
-export const getRecord = async (builder, config, accessId) => {
+export const getRecord = async (builder, accessIdWithoutLimit, episode = 0) => {
     await awaitStore();
+    const { io } = Scoped.ReleaseCacheData;
     const { projectUrl, dbUrl, dbName, path, command } = builder;
-    const { find, findOne, sort, direction, limit, random } = command;
-    const entityId = await generateRecordID({}, config);
-    const colData = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'data', entityId]);
-    const colRecord = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'record', accessId]);
-
-    if (!colRecord) return null;
-    let choosenColData = colData.listing.filter(v =>
-        !colRecord.ignore.includes(v._id) &&
-        confirmFilterDoc(v, findOne || find || {})
-    );
+    const { limit, sort, direction, random, findOne } = command;
+    const isEpisode = episode === 1;
+
+    const transformData = (data) => {
+        data = cloneDeep(data);
+        if (random) {
+            data = shuffleArray(data);
+        } else if (sort) {
+            data = sortArrayByObjectKey(data.slice(0), sort);
+            if (
+                direction === -1 ||
+                direction === 'desc' ||
+                direction === 'descending'
+            ) data = data.slice(0).reverse();
+        }
 
-    if (random) {
-        choosenColData = shuffleArray(choosenColData);
-    } else if (sort) {
-        sortArrayByObjectKey(choosenColData, sort);
-        if (
-            direction === -1 ||
-            direction === 'desc' ||
-            direction === 'descending'
-        ) choosenColData.reverse();
+        if (findOne) {
+            data = data[0];
+        } else if (limit) data = data.slice(0, limit);
+
+        return data;
     }
 
-    if (findOne) {
-        choosenColData = choosenColData[0];
-    } else if (limit) choosenColData.slice(0, limit);
+    if (!io) {
+        const record = await useSqliteLinearAccessId(builder, accessIdWithoutLimit, 'database')(async sqlite => {
+            const resultAccessId = `${accessIdWithoutLimit}-${limit}`;
 
-    return [choosenColData, colRecord.result];
+            const qData = await (
+                isEpisode ? sqlite.executeSql(`SELECT * FROM ${LIMITER_RESULT(path)} WHERE access_id_limiter = ?`, [resultAccessId]) :
+                    sqlite.executeSql(`SELECT * FROM ${LIMITER_DATA(path)} WHERE access_id = ?`, [accessIdWithoutLimit])
+            ).then(v => v[0].rows.item(0)).catch(() => null);
+            const thisData = qData && deserializeBSON(await parseBigData(qData.value), true);
+
+            if (!thisData) return null;
+
+            if (isEpisode) {
+                await sqlite.executeSql(SQLITE_COMMANDS.UPDATE_COLUMNS(LIMITER_RESULT(path), ['touched'], 'access_id_limiter = ?'), [Date.now(), resultAccessId]);
+                return [thisData.data];
+            }
+
+            const { latest_limiter, data } = thisData;
+
+            if (
+                latest_limiter === undefined ||
+                (Validator.POSITIVE_NUMBER(limit) && latest_limiter >= limit)
+            ) {
+                await sqlite.executeSql(SQLITE_COMMANDS.UPDATE_COLUMNS(LIMITER_DATA(path), ['touched'], 'access_id = ?'), [Date.now(), accessIdWithoutLimit]);
+                return [transformData(data)];
+            }
+        });
+
+        return record || null;
+    }
+
+    if (isEpisode) {
+        const resultData = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'episode', accessIdWithoutLimit, limit]);
+        if (resultData) {
+            resultData.touched = Date.now();
+            return [cloneDeep(resultData.data)];
+        }
+        return null;
+    }
+
+    const instanceData = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'instance', accessIdWithoutLimit]);
+    if (!instanceData) return null;
+    const { latest_limiter, data } = instanceData;
+
+    if (
+        latest_limiter === undefined ||
+        (Validator.POSITIVE_NUMBER(limit) && latest_limiter >= limit)
+    ) {
+        instanceData.touched = Date.now();
+        return [transformData(data)];
+    }
+
+    return null;
 };
 
-export const generateRecordID = (builder, config) => {
+export const generateRecordID = (builder, config, removeLimit) => {
     builder = builder && cloneDeep(builder);
     config = config && cloneDeep(config);
 
@@ -151,16 +303,16 @@ export const generateRecordID = (builder, config) => {
         }).filter(([_, v]) => v !== undefined)
     );
 
-    if (command) recordObj.command = arrangeCommands(command);
+    if (command) recordObj.command = arrangeCommands(command, removeLimit);
     if (extraction) {
-        if (Array.isArray(extraction)) recordObj.extraction = extraction.map(arrangeCommands);
+        if (Array.isArray(extraction)) recordObj.extraction = extraction.map(v => arrangeCommands(v));
         else recordObj.extraction = arrangeCommands(extraction);
     }
 
     return niceHash(serializeToBase64(recordObj));
 };
 
-const arrangeCommands = c => {
+const arrangeCommands = (c, removeLimit) => {
     c = cloneDeep(c);
     const sortFind = f => {
         ['$and', '$or', '$nor'].forEach(n => {
@@ -174,11 +326,12 @@ const arrangeCommands = c => {
     if (c.sort) c.direction = [-1, 'desc', 'descending'].includes(c.direction) ? 'desc' : 'asc';
     if (c.find) c.find = sortFind(c.find);
     if (c.findOne) c.findOne = sortFind(c.findOne);
+    if (removeLimit && 'limit' in c) delete c.limit;
     return sortObject(c);
 };
 
 const sortObject = (o) => Object.fromEntries(
-    Object.entries(o).sort((a, b) => (a > b) ? 1 : (a < b) ? -1 : 0)
+    Object.entries(o).sort(([a], [b]) => (a > b) ? 1 : (a < b) ? -1 : 0)
 );
 
 const recursiveFlat = (a) => {
@@ -270,31 +423,98 @@ export const addPendingWrites = async (builder, writeId, result) => {
     result = result && cloneDeep(result);
     await awaitStore();
 
+    const { io } = Scoped.ReleaseCacheData;
     const { projectUrl, dbUrl, dbName } = builder;
     const editions = [];
     const duplicateSets = {};
     const pathChanges = new Set([]);
+    const pendingSnapshot = cloneDeep(result);
 
-    (
+    await Promise.all((
         result.type === 'batchWrite' ?
             result.value.map(({ scope, value, find, path }) =>
                 ({ type: scope, value, find, path })
             )
             : [{ ...result, path: builder.path }]
-    ).forEach(({ value: writeObj, find, type, path }) => {
+    ).map(async ({ value: writeObj, find, type, path }) => {
         WriteValidator[type]({ find, value: writeObj });
         validateCollectionName(path);
         pathChanges.add(path);
-        const colObj = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'data'], {});
 
-        Object.entries(colObj).forEach(([entityId, { listing, config }]) => {
+        if (io) {
+            const colObj = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'instance']);
+
+            if (colObj)
+                await Promise.all(
+                    Object.entries(colObj).map(e =>
+                        MutateDataInstance(
+                            e,
+                            path =>
+                                Object.values(
+                                    getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'instance'], {})
+                                ).map(({ data }) => data).flat()
+                        )
+                    )
+                );
+        } else {
+            const sqlite = await openDB(builder);
+            try {
+                const colListing = await sqlite.executeSql(`SELECT access_id FROM ${LIMITER_DATA(path)}`).then(v =>
+                    v[0].rows.raw().map(d => d.access_id)
+                ).catch(() => []);
+                const pathFinder = {};
+
+                await Promise.all(colListing.map(async access_id =>
+                    useSqliteLinearAccessId(builder, access_id, 'database')(async (sqlite, db_filename) => {
+                        const data = await sqlite.executeSql(`SELECT * FROM ${LIMITER_DATA(path)} WHERE access_id = ?`, [access_id]).then(async v => {
+                            const d = v[0].rows.item(0);
+                            return [d.access_id, deserializeBSON(await parseBigData(d.value), true)];
+                        });
+                        await MutateDataInstance(data, path =>
+                            pathFinder[path] || (
+                                pathFinder[path] = sqlite.executeSql(`SELECT value FROM ${LIMITER_DATA(path)}`).then(v =>
+                                    Promise.all(
+                                        v[0].rows.raw().map(async d =>
+                                            deserializeBSON(await parseBigData(d.value), true).data
+                                        )
+                                    ).then(r => r.flat())
+                                ).catch(() => [])
+                            )
+                        );
+                        const editedBlobData = await handleBigData(
+                            getStoreID(db_filename, LIMITER_DATA(path), access_id),
+                            serializeToBase64(data[1])
+                        );
+
+                        await sqlite.executeSql(
+                            SQLITE_COMMANDS.MERGE(LIMITER_DATA(path), ['access_id', 'value', 'touched', 'size']),
+                            [access_id, editedBlobData, Date.now(), data[1].size]
+                        );
+                    })
+                ));
+            } catch (error) {
+                throw error;
+            } finally {
+                sqlite.close();
+            }
+        }
+
+        async function MutateDataInstance([entityId, dataObj], pathGetter) {
+            const { data: instance_data, command, config } = dataObj;
+            const entityFind = command.findOne || command.find;
             const { extraction } = config || {};
 
             const logChanges = (d) => {
-                editions.push([entityId, d, path]);
+                editions.push(cloneDeep([entityId, d, path]));
+                const [b4, af] = d;
+                const offset = docSize(af) - docSize(b4);
+                dataObj.size += offset;
+                incrementDatabaseSize(builder, path, offset);
             };
 
-            const accessExtraction = obj => {
+            const snipUpdate = doc => snipDocument(doc, entityFind, config);
+
+            const accessExtraction = async obj => {
                 const buildAssignedExtraction = (data) => {
                     const d = (Array.isArray(extraction) ? extraction : [extraction]).map(thisExtraction => {
                         const query = cloneDeep(thisExtraction);
@@ -311,18 +531,18 @@ export const addPendingWrites = async (builder, writeId, result) => {
                 const extractionResultant = buildAssignedExtraction(obj);
                 const extractionBinary = serializeToBase64({ _: extractionResultant });
 
-                const sameProjection = listing.find(({ _foreign_doc, ...restDoc }) =>
+                const sameProjection = instance_data.find(({ _foreign_doc, ...restDoc }) =>
                     extractionBinary === serializeToBase64({ _: buildAssignedExtraction(restDoc) })
                 );
 
                 if (sameProjection) return sameProjection._foreign_doc;
 
-                // if no matching extraction was found, proceed to scrapping other paths
-                const scrapedProjection = (Array.isArray(extractionResultant) ? extractionResultant : [extractionResultant]).map((query, i) => {
+                // if no matching extraction was found, proceed to scrapping each _foreign_doc segment
+                const scrapedProjection = await Promise.all((Array.isArray(extractionResultant) ? extractionResultant : [extractionResultant]).map(async (query, i) => {
                     const { sort, direction, limit, find, findOne, collection: path } = query;
-                    const scrapDocs = [];
+                    let scrapDocs = [];
 
-                    listing.forEach(({ _foreign_doc }) => {
+                    instance_data.forEach(({ _foreign_doc }) => {
                         _foreign_doc = (Array.isArray(_foreign_doc) ? _foreign_doc : [_foreign_doc])[i];
 
                         recursiveFlat([_foreign_doc]).forEach(e => {
@@ -331,83 +551,102 @@ export const addPendingWrites = async (builder, writeId, result) => {
                             }
                         });
                     });
+
                     if (!scrapDocs.length) {
-                        const scrapYard = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'data'], {});
-                        Object.values(scrapYard).forEach(v => {
-                            v.listing.forEach(doc => {
-                                if (confirmFilterDoc(doc, find || findOne)) {
-                                    scrapDocs.push(snipDocument(doc, find || findOne, config));
-                                }
-                            });
+                        // if no matching extraction was found, proceed to scrapping ancestor path
+                        (await pathGetter(path)).forEach(({ _foreign_doc, ...doc }) => {
+                            if (confirmFilterDoc(doc, find || findOne)) {
+                                scrapDocs.push(doc);
+                            }
                         });
                     }
+                    scrapDocs = scrapDocs.filter((v, i, a) => a.findIndex(b => b._id === v._id) === i);
                     if (sort) sortArrayByObjectKey(scrapDocs, sort);
                     if ([-1, 'desc', 'descending'].includes(direction)) scrapDocs.reverse();
-                    if (limit) scrapDocs.splice(limit);
+                    if (limit) scrapDocs = scrapDocs.slice(0, limit);
+                    scrapDocs = scrapDocs.map(v => snipDocument(v, find || findOne, query));
 
                     return findOne ? scrapDocs[0] : scrapDocs;
-                });
+                }));
 
-                return Array.isArray(extraction) ? scrapedProjection : scrapedProjection[0];
+                return cloneDeep(Array.isArray(extraction) ? scrapedProjection : scrapedProjection[0]);
             }
 
             if (['setOne', 'setMany'].includes(type)) {
-                (type === 'setOne' ? [writeObj] : writeObj).forEach(e => {
-                    if (listing.findIndex(v => CompareBson.equal(v._id, e._id)) === -1) {
-                        const obj = deserializeNonAtomicWrite(e);
-
-                        if (extraction) obj._foreign_doc = accessExtraction(obj);
-                        listing.push(obj);
-                        logChanges([undefined, obj]);
-                    } else if (!duplicateSets[e._id])
-                        console.warn(`document with _id=${e._id} already exist locally with ${type}() operation, skipping to online commit`);
-                    duplicateSets[e._id] = true;
-                });
+                await Promise.all((type === 'setOne' ? [writeObj] : writeObj).map(async e => {
+                    const obj = deserializeNonAtomicWrite(e);
+                    if (extraction) obj._foreign_doc = await accessExtraction(obj);
+
+                    if (confirmFilterDoc(obj, entityFind)) {
+
+                        if (instance_data.findIndex(v => CompareBson.equal(v._id, e._id)) === -1) {
+                            const x = snipUpdate(obj);
+                            instance_data.push(cloneDeep(x));
+                            logChanges([undefined, x]);
+                        } else if (!duplicateSets[e._id]) {
+                            console.warn(`document with _id=${e._id} already exist locally with ${type}() operation, skipping to online commit`);
+                            duplicateSets[e._id] = true;
+                        }
+                    }
+                }));
                 return;
             }
 
             if (['putOne', 'replaceOne'].includes(type)) {
                 const extras = createWriteFromFind(find);
 
-                for (let i = 0; i < listing.length; i++) {
-                    const doc = listing[i];
+                let deletions = 0;
+                const cdata = instance_data.slice(0);
+
+                for (let i = 0; i < cdata.length; i++) {
+                    const doc = cdata[i];
+
                     if (confirmFilterDoc(doc, find)) {
                         const obj = deserializeNonAtomicWrite({
                             ...extras,
                             ...writeObj,
                             ...'_id' in extras ? {} : { _id: doc._id }
                         });
-
-                        if (extraction) obj._foreign_doc = accessExtraction(obj);
-                        listing[i] = obj;
-                        logChanges([doc, obj]);
+                        if (extraction) obj._foreign_doc = await accessExtraction(obj);
+
+                        if (confirmFilterDoc(obj, entityFind)) {
+                            const x = snipUpdate(obj);
+                            instance_data[i - deletions] = x;
+                            logChanges([doc, x]);
+                        } else {
+                            instance_data.splice(i - deletions++, 1);
+                            logChanges([doc, undefined]);
+                        }
                         return;
                     }
                 }
+
                 if (type === 'putOne') {
                     const obj = deserializeNonAtomicWrite({
                         ...extras,
                         ...writeObj,
                         ...'_id' in extras ? {} : { _id: new ObjectId() }
                     });
+                    if (extraction) obj._foreign_doc = await accessExtraction(obj);
 
-                    if (extraction) obj._foreign_doc = accessExtraction(obj);
-                    listing.push(obj);
-                    logChanges([undefined, obj]);
+                    if (confirmFilterDoc(obj, entityFind)) {
+                        const x = snipUpdate(obj);
+                        instance_data.push(x);
+                        logChanges([undefined, x]);
+                    }
                 }
                 return;
             }
 
             if (['deleteOne', 'deleteMany'].includes(type)) {
                 let deletions = 0;
+                const cdata = instance_data.slice(0);
 
-                for (let i = 0; i < listing.length; i++) {
-                    const dex = deletions + i;
-                    const doc = listing[dex];
+                for (let i = 0; i < cdata.length; i++) {
+                    const doc = cdata[i];
                     if (confirmFilterDoc(doc, find)) {
-                        listing.splice(dex, 1);
-                        logChanges([doc]);
-                        --deletions;
+                        instance_data.splice(i - deletions++, 1);
+                        logChanges([doc, undefined]);
                         if (type === 'deleteOne') return;
                     }
                 }
@@ -415,14 +654,23 @@ export const addPendingWrites = async (builder, writeId, result) => {
             }
 
             let founded;
-            for (let i = 0; i < listing.length; i++) {
-                const doc = listing[i];
+            let deletions = 0;
+            const cdata = instance_data.slice(0);
+
+            for (let i = 0; i < cdata.length; i++) {
+                const doc = cdata[i];
                 if (confirmFilterDoc(doc, find)) {
                     const obj = deserializeAtomicWrite(doc, deserializeWriteValue(writeObj), false, type);
-
-                    if (extraction) obj._foreign_doc = accessExtraction(obj);
-                    listing[i] = obj;
-                    logChanges([doc, obj]);
+                    if (extraction) obj._foreign_doc = await accessExtraction(obj);
+
+                    if (confirmFilterDoc(obj, entityFind)) {
+                        const x = snipUpdate(obj);
+                        instance_data[i - deletions] = x;
+                        logChanges([doc, x]);
+                    } else {
+                        instance_data.splice(i - deletions++, 1);
+                        logChanges([doc, undefined]);
+                    }
 
                     founded = true;
                     if (type.endsWith('One')) return;
@@ -440,22 +688,25 @@ export const addPendingWrites = async (builder, writeId, result) => {
                         type
                     )
                 };
+                if (extraction) obj._foreign_doc = await accessExtraction(obj);
 
-                if (extraction) obj._foreign_doc = accessExtraction(obj);
-                listing.push(obj);
-                logChanges([undefined, obj]);
+                if (confirmFilterDoc(obj, entityFind)) {
+                    const x = snipUpdate(obj);
+                    instance_data.push(x);
+                    logChanges([undefined, x]);
+                }
             }
-        });
-    });
+        };
+    }));
 
     setLodash(CacheStore.PendingWrites, [projectUrl, writeId], cloneDeep({
         builder,
-        snapshot: result,
+        snapshot: pendingSnapshot,
         editions,
         addedOn: Date.now()
     }));
 
-    updateCacheStore();
+    updateCacheStore(undefined, ['DatabaseStore', 'PendingWrites', 'DatabaseStats']);
     notifyDatabaseNodeChanges(builder, [...pathChanges]);
 };
 
@@ -463,40 +714,75 @@ export const removePendingWrite = async (builder, writeId, revert) => {
     await awaitStore();
     const { projectUrl, dbUrl, dbName } = builder;
     const pendingData = getLodash(CacheStore.PendingWrites, [projectUrl, writeId]);
+    const { io } = Scoped.ReleaseCacheData;
 
     if (!pendingData) return;
     const pathChanges = new Set([]);
 
     if (revert) {
-        pendingData.editions.forEach(([entityId, [b4Doc, afDoc], path]) => {
-            const colObj = getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'data']);
-            const colList = colObj?.[entityId]?.listing;
-
-            if (colList) {
-                if (afDoc) {
-                    const editedIndex = colList.findIndex(e => CompareBson.equal(e._id, afDoc._id));
-                    if (editedIndex !== -1) {
-                        if (
-                            serializeToBase64(afDoc) === serializeToBase64(colList[editedIndex])
-                        ) {
-                            if (b4Doc) {
-                                colList[editedIndex] = b4Doc;
-                            } else colList.splice(editedIndex, 1);
+        await Promise.all(pendingData.editions.map(async ([access_id, [b4Doc, afDoc], path]) => {
+            if (io) {
+                RevertMutation(getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'instance', access_id]));
+            } else {
+                await useSqliteLinearAccessId(builder, access_id, 'database')(async (sqlite, db_filename) => {
+                    const colObj = await sqlite.executeSql(`SELECT * FROM ${LIMITER_DATA(path)} WHERE access_id = ?`, [access_id]).then(async v => {
+                        const d = v[0].rows.item(0);
+                        return deserializeBSON(await parseBigData(d.value));
+                    }).catch(() => null);
+                    if (!colObj) return;
+                    RevertMutation(colObj);
+                    const revertedBlobData = await handleBigData(
+                        getStoreID(db_filename, LIMITER_DATA(path), access_id),
+                        serializeToBase64(colObj)
+                    );
+
+                    await sqlite.executeSql(
+                        SQLITE_COMMANDS.MERGE(LIMITER_DATA(path), ['access_id', 'value', 'touched', 'size']),
+                        [access_id, revertedBlobData, Date.now(), colObj.size]
+                    );
+                });
+            }
+
+            function RevertMutation(colObj) {
+                const colList = colObj?.data;
+
+                const updateSize = (b4, af) => {
+                    const offset = docSize(af) - docSize(b4);
+                    colObj.size += offset;
+                    incrementDatabaseSize(builder, path, offset);
+                }
+
+                if (colList) {
+                    if (afDoc) {
+                        const editedIndex = colList.findIndex(e => CompareBson.equal(e._id, afDoc._id));
+                        if (editedIndex !== -1) {
+                            if (
+                                serializeToBase64(afDoc) === serializeToBase64(colList[editedIndex])
+                            ) {
+                                if (b4Doc) {
+                                    colList[editedIndex] = b4Doc;
+                                    updateSize(afDoc, b4Doc);
+                                } else {
+                                    colList.splice(editedIndex, 1);
+                                    updateSize(afDoc, undefined);
+                                }
+                            }
                         }
+                    } else if (
+                        b4Doc &&
+                        colList.findIndex(e => CompareBson.equal(e._id, b4Doc._id)) === -1
+                    ) {
+                        colList.push(b4Doc);
+                        updateSize(undefined, b4Doc);
                     }
-                } else if (
-                    b4Doc &&
-                    colList.findIndex(e => CompareBson.equal(e._id, b4Doc._id)) === -1
-                ) {
-                    colList.push(b4Doc);
                 }
+                pathChanges.add(path);
             }
-            pathChanges.add(path);
-        });
+        }));
     }
 
     unsetLodash(CacheStore.PendingWrites, [projectUrl, writeId]);
-    updateCacheStore();
+    updateCacheStore(undefined, ['PendingWrites', 'DatabaseStore', 'DatabaseStats']);
     notifyDatabaseNodeChanges(builder, [...pathChanges]);
 };
 
@@ -541,7 +827,7 @@ const snipDocument = (data, find, config) => {
     if (!data || !config) return data;
     const { returnOnly, excludeFields } = config || {};
 
-    let output = { ...data };
+    let output = cloneDeep(data);
 
     if (returnOnly) {
         output = {};
@@ -601,7 +887,6 @@ const deserializeWriteValue = (value) => {
 
 const deserializeNonAtomicWrite = (writeObj) => deserializeWriteValue(writeObj);
 
-
 const deserializeAtomicWrite = (b4Doc, writeObj, isNew, type) => {
     const resultantDoc = { ...b4Doc };