react-native-mosquito-transport 0.0.35 → 0.0.37

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-mosquito-transport",
-  "version": "0.0.35",
+  "version": "0.0.37",
   "description": "React native javascript sdk for mosquito-transport (https://github.com/brainbehindx/mosquito-transport)",
   "main": "src/index.js",
   "type": "module",

package/src/helpers/fs_manager.js

@@ -1,6 +1,8 @@
+import { deserialize, serialize } from "entity-serializer";
 import { Scoped } from "./variables";
 import { Platform } from "react-native";
 import { Dirs, FileSystem } from "react-native-file-access";
+import { Buffer } from "buffer";
 
 const PARENT_FOLDER = `${Platform.OS === 'android' ? Dirs.DocumentDir.split('/').slice(0, -1).join('/') : Dirs.MainBundleDir}/mosquito_base`;
 
@@ -48,7 +50,7 @@ export const getSystem = (builder) => {
             const path = conjoin(table, primary_key);
             await FileSystem.mkdir(path).catch(() => null);
             await Promise.all(Object.entries(value).map(([k, v]) =>
-                FileSystem.writeFile(joinPath(path, k), JSON.stringify(v), 'utf8')
+                FileSystem.writeFile(joinPath(path, k), serialize(v).toString('base64'), 'base64')
             ));
         },
         delete: (table, primary_key) => FileSystem.unlink(conjoin(table, primary_key)),
@@ -56,7 +58,7 @@ export const getSystem = (builder) => {
             const path = conjoin(table, primary_key);
 
             const value_map = await Promise.all(extractions.map(async node =>
-                [node, JSON.parse(await FileSystem.readFile(joinPath(path, node), 'utf8'))]
+                [node, deserialize(Buffer.from(await FileSystem.readFile(joinPath(path, node), 'base64'), 'base64'))]
             ));
             return Object.fromEntries(value_map);
         },

package/src/helpers/purger.js

@@ -12,7 +12,7 @@ const inlineFsData = (arr, access_id_node = 'access_id') =>
         return obj;
     });
 
-export const purgeRedundantRecords = async (data, builder) => {
+export const purgeRedundantRecords = async (data, builder, willPurge) => {
     const { io, maxLocalDatabaseSize = 10485760, maxLocalFetchHttpSize = 10485760 } = builder;
 
     /**
@@ -20,9 +20,15 @@ export const purgeRedundantRecords = async (data, builder) => {
      */
     const { _db_size, _fetcher_size, counters, database, fetchers } = data.DatabaseStats || {};
 
+    const shouldPurgeDb = Validator.POSITIVE_NUMBER(_db_size) && maxLocalDatabaseSize && _db_size >= maxLocalDatabaseSize;
+    const shouldPurgeFetcher = Validator.POSITIVE_NUMBER(_fetcher_size) && maxLocalFetchHttpSize && _fetcher_size >= maxLocalFetchHttpSize;
+
+    if (shouldPurgeDb) willPurge(['DatabaseStore', 'DatabaseCountResult', 'DatabaseStats']);
+    if (shouldPurgeFetcher) willPurge(['FetchedStore', 'DatabaseStats']);
+
     if (io) {
         const purgeDatabase = () => {
-            if (!Validator.POSITIVE_NUMBER(_db_size) || !maxLocalDatabaseSize || _db_size < maxLocalDatabaseSize) return;
+            if (!shouldPurgeDb) return;
             const DbListing = [];
 
             breakDbMap(data.DatabaseStore, (projectUrl, dbUrl, dbName, path, value) => {
@@ -98,7 +104,7 @@ export const purgeRedundantRecords = async (data, builder) => {
             });
         }
         const purgeFetcher = () => {
-            if (!Validator.POSITIVE_NUMBER(_fetcher_size) || !maxLocalFetchHttpSize || _fetcher_size < maxLocalFetchHttpSize) return;
+            if (!shouldPurgeFetcher) return;
             const redundantFetchRanking = Object.entries(data.FetchedStore).map(([projectUrl, access_id_Obj]) =>
                 Object.entries(access_id_Obj).map(([access_id, data]) => ({
                     access_id,
@@ -133,7 +139,7 @@ export const purgeRedundantRecords = async (data, builder) => {
         await Promise.allSettled([
             (async () => {
                 try {
-                    if (!Validator.POSITIVE_NUMBER(_db_size) || !maxLocalDatabaseSize || _db_size < maxLocalDatabaseSize) return;
+                    if (!shouldPurgeDb) return;
                     const instances = [];
 
                     [database, counters].forEach((map, i) => {
@@ -204,7 +210,7 @@ export const purgeRedundantRecords = async (data, builder) => {
             })(),
             (async () => {
                 try {
-                    if (!Validator.POSITIVE_NUMBER(_fetcher_size) || !maxLocalFetchHttpSize || _fetcher_size < maxLocalFetchHttpSize) return;
+                    if (!shouldPurgeFetcher) return;
 
                     const redundantFetchRanking = await Promise.all(
                         Object.entries(fetchers).map(async ([projectUrl]) => {
@@ -245,7 +251,7 @@ export const purgeRedundantRecords = async (data, builder) => {
     }
 }
 
-const breakDbMap = (obj, callback) =>
+export const breakDbMap = (obj = {}, callback) =>
     Object.entries(obj).forEach(([projectUrl, dbUrlObj]) => {
         Object.entries(dbUrlObj).forEach(([dbUrl, dbNameObj]) => {
             Object.entries(dbNameObj).forEach(([dbName, pathObj]) => {

package/src/helpers/utils.js

@@ -1,16 +1,46 @@
 import { ServerReachableListener, StoreReadyListener } from "./listeners";
 import { CacheStore, Scoped } from "./variables";
 import { serializeE2E } from "./peripherals";
-import { deserializeBSON, serializeToBase64 } from "../products/database/bson";
+import { DatastoreParser } from "../products/database/bson";
 import { deserialize } from "entity-serializer";
-import { purgeRedundantRecords } from "./purger";
+import { breakDbMap, purgeRedundantRecords } from "./purger";
 import { FS_PATH, getSystem } from "./fs_manager";
+import cloneDeep from "lodash/cloneDeep";
+import { Buffer } from "buffer";
 
 const { FILE_NAME, TABLE_NAME } = FS_PATH;
 
 const CacheKeys = Object.keys(CacheStore);
 
+const prefillDatastore = (obj, caller) => {
+    obj = cloneDeep(obj);
+    breakDbMap(obj, (_projectUrl, _dbUrl, _dbName, _path, value) => {
+        Object.entries(value.instance).forEach(([access_id, obj]) => {
+            value.instance[access_id] = caller(obj);
+        });
+        Object.entries(value.episode).forEach(([_access_id, limitObj]) => {
+            Object.entries(limitObj).forEach(([limit, obj]) => {
+                limitObj[limit] = caller(obj);
+            });
+        });
+    });
+    return obj;
+};
+
+const prefillFetcher = (store, encode) => {
+    store = cloneDeep(store);
+    Object.values(store).forEach(accessIdObj => {
+        Object.values(accessIdObj).forEach(value => {
+            value.data.buffer = encode ?
+                Buffer.from(value.data.buffer).toString('base64')
+                : Buffer.from(value.data.buffer, 'base64');
+        });
+    });
+    return store;
+}
+
 export const updateCacheStore = async (node) => {
+    node = node && node.filter((v, i, a) => a.indexOf(v) === i);
     const { io, promoteCache } = Scoped.ReleaseCacheData;
 
     const {
@@ -19,17 +49,29 @@ export const updateCacheStore = async (node) => {
         PendingAuthPurge,
         DatabaseStore,
         PendingWrites,
+        FetchedStore,
         ...restStore
     } = CacheStore;
 
+    const minimizePendingWrite = () => {
+        const obj = cloneDeep(PendingWrites);
+        Object.values(obj).forEach(e => {
+            Object.values(e).forEach(b => {
+                if ('editions' in b) delete b.editions;
+            });
+        });
+        return obj;
+    }
+
     if (io) {
         const txt = JSON.stringify({
             AuthStore,
             EmulatedAuth,
             PendingAuthPurge,
             ...promoteCache ? {
-                DatabaseStore: serializeToBase64(DatabaseStore),
-                PendingWrites: serializeToBase64(PendingWrites)
+                DatabaseStore: prefillDatastore(DatabaseStore, DatastoreParser.encode),
+                PendingWrites: minimizePendingWrite(),
+                FetchedStore: prefillFetcher(FetchedStore, true)
             } : {},
             ...promoteCache ? restStore : {}
         });
@@ -43,7 +85,7 @@ export const updateCacheStore = async (node) => {
         if (!updationKey.length) return;
         await Promise.all(
             updationKey
-                .map(v => [v, v === 'PendingWrites' ? serializeToBase64(CacheStore[v]) : CacheStore[v]])
+                .map(v => [v, v === 'PendingWrites' ? minimizePendingWrite() : CacheStore[v]])
                 .map(([ref, value]) =>
                     getSystem(FILE_NAME).set(TABLE_NAME, ref, { value })
                 )
@@ -57,37 +99,45 @@ export const releaseCacheStore = async (builder) => {
     const { io } = builder;
 
     let data = {};
+    const tobePurged = [];
 
     try {
         if (io) {
             data = JSON.parse((await io.input()) || '{}');
-        } else {
-            try {
-                const query = await getSystem(FILE_NAME).list(TABLE_NAME, ['value']).catch(() => []);
-                data = Object.fromEntries(
-                    query.map(([ref, { value }]) =>
-                        [ref, value]
-                    )
+
+            if (data.DatabaseStore)
+                data.DatabaseStore = prefillDatastore(
+                    data.DatabaseStore,
+                    r => DatastoreParser.decode(r, false)
                 );
-            } catch (error) {
-                console.error('initializeCache data release err:', error);
-            }
+            if (data.FetchedStore)
+                data.FetchedStore = prefillFetcher(data.FetchedStore, false);
+        } else {
+            const query = await getSystem(FILE_NAME).list(TABLE_NAME, ['value']).catch(() => []);
+            data = Object.fromEntries(
+                query.map(([ref, { value }]) =>
+                    [ref, value]
+                )
+            );
         }
-        await purgeRedundantRecords(data, builder);
+        await purgeRedundantRecords(data, builder, purgeNodes => {
+            tobePurged.push(...purgeNodes);
+        });
     } catch (e) {
-        console.error('initializeCache data err:', e);
+        console.error('releaseCacheStore data err:', e);
     }
 
     Object.entries(data).forEach(([k, v]) => {
-        if (['DatabaseStore', 'PendingWrites'].includes(k)) {
-            CacheStore[k] = deserializeBSON(v);
-        } else CacheStore[k] = v;
+        CacheStore[k] = v;
     });
     Object.entries(CacheStore.AuthStore).forEach(([key, value]) => {
         Scoped.AuthJWTToken[key] = value?.token;
     });
     Scoped.IsStoreReady = true;
     StoreReadyListener.dispatch('_', 'ready');
+    setTimeout(() => {
+        if (tobePurged.length) updateCacheStore(tobePurged);
+    }, 0);
 };
 
 export const getPrefferTime = () => Date.now() + (Scoped.serverTimeOffset || 0);

package/src/helpers/values.js

@@ -12,7 +12,6 @@ export const RETRIEVAL = {
 
 export const DELIVERY = {
     DEFAULT: 'default',
-    CACHE_AWAIT: 'cache-await', // TODO: remove
     CACHE_NO_AWAIT: 'cache-no-await',
     NO_CACHE_NO_AWAIT: 'no-cache-no-await',
     NO_CACHE_AWAIT: 'no-cache-await'

package/src/index.d.ts

@@ -1,3 +1,5 @@
+import { BSON } from "./vendor/bson";
+
 interface RNMTConfig {
     dbName?: string;
     dbUrl?: string;
@@ -75,6 +77,8 @@ export class DoNotEncrypt {
     value: any;
 }
 
+export { BSON }
+
 interface auth_provider_id {
     GOOGLE: 'google';
     FACEBOOK: 'facebook';
@@ -289,7 +293,7 @@ interface FetchHttpConfig {
     rawApproach?: boolean;
 }
 
-type Delievery = 'default' | 'no-cache' | 'no-await' | 'no-await-no-cache' | 'await-no-cache' | 'cache-no-await';
+type Delievery = 'default' | 'cache-no-await' | 'no-cache-no-await' | 'no-cache-await';
 
 interface WriteConfig {
     /**
@@ -297,21 +301,9 @@ interface WriteConfig {
      */
     disableAuth?: boolean;
 
-    DEFAULT: 'default',
-    CACHE_AWAIT: 'cache-await',
-    CACHE_NO_AWAIT: 'cache-no-await',
-    NO_CACHE_NO_AWAIT: 'no-cache-no-await',
-    NO_CACHE_AWAIT: 'no-cache-await'
-
     /**
      * This property determines how the write will be dispatch to the remote server
      *  TODO:
-     * 
-     * `default`: 
-     * `cache-await`: 
-     * `cache-no-await`: 
-     * `no-cache-no-await`: 
-     * `no-cache-await`: 
      */
     delivery?: Delievery;
 }

package/src/index.js

@@ -15,6 +15,7 @@ import { Validator } from 'guard-object';
 import cloneDeep from 'lodash/cloneDeep';
 import { Buffer } from 'buffer';
 import MTAuth, { purgePendingToken } from './products/auth';
+import { BSON } from "./vendor/bson";
 
 const {
     _listenCollection,
@@ -426,7 +427,7 @@ const validateReleaseCacheProp = (prop) => {
                         throw `Invalid value supplied to "io.${k}", expected a function but got "${v}"`;
                 } else throw `Unexpected property named "io.${k}"`;
             });
-            if (!v?.input && !v?.output) throw '"input" and "output" are required when "io" is provided';
+            if (!v?.input || !v?.output) throw '"input" and "output" are required when "io" is provided';
         } else if (k === 'promoteCache') {
             if (typeof v !== 'boolean') throw 'promoteCache should be a boolean';
         } else if (['maxLocalDatabaseSize', 'maxLocalFetchHttpSize'].includes(k)) {
@@ -507,7 +508,8 @@ export {
     DOCUMENT_EXTRACTION,
     FIND_GEO_JSON,
     GEO_JSON,
-    AUTH_PROVIDER_ID
+    AUTH_PROVIDER_ID,
+    BSON
 };
 
 export default RNMT;

package/src/products/database/accessor.js

@@ -6,9 +6,9 @@ import { DatabaseRecordsListener } from "../../helpers/listeners";
 import cloneDeep from "lodash/cloneDeep";
 import { BSONRegExp, ObjectId, Timestamp } from "../../vendor/bson";
 import { niceGuard, Validator } from "guard-object";
-import { TIMESTAMP } from "../..";
+import { TIMESTAMP } from "./types";
 import { docSize, incrementDatabaseSize } from "./counter";
-import { deserializeBSON, serializeToBase64 } from "./bson";
+import { DatastoreParser, serializeToBase64 } from "./bson";
 import { FS_PATH, getSystem, useFS } from "../../helpers/fs_manager";
 import { grab, poke, unpoke } from "poke-object";
 
@@ -100,14 +100,13 @@ export const insertRecord = async (builder, config, accessIdWithoutLimit, value,
             const editionSizeOffset = thisSize - (instanceData?.size || 0);
             const resultSizeOffset = isEpisode ? thisSize - (resultData?.size || 0) : 0;
 
-            const newData = serializeToBase64({
+            const newData = DatastoreParser.encode({
                 command,
                 config,
                 latest_limiter: limit,
-                size: thisSize,
                 data: value ? Array.isArray(value) ? value : [value] : []
             });
-            const newResultData = isEpisode && serializeToBase64({
+            const newResultData = isEpisode && DatastoreParser.encode({
                 data: value,
                 size: thisSize
             });
@@ -195,7 +194,7 @@ export const getRecord = async (builder, accessIdWithoutLimit, episode = 0) => {
                 isEpisode ? fs.find(LIMITER_RESULT(path), resultAccessId, ['value']) :
                     fs.find(LIMITER_DATA(path), accessIdWithoutLimit, ['value'])
             ).catch(() => null);
-            const thisData = qData && deserializeBSON(qData.value, true);
+            const thisData = qData && DatastoreParser.decode(qData.value);
 
             if (!thisData) return null;
 
@@ -283,7 +282,7 @@ const arrangeCommands = (c, removeLimit) => {
     if (c.sort) c.direction = [-1, 'desc', 'descending'].includes(c.direction) ? 'desc' : 'asc';
     if (c.find) c.find = sortFind(c.find);
     if (c.findOne) c.findOne = sortFind(c.findOne);
-    if (removeLimit && 'limit' in c) delete c.limit;
+    if (removeLimit && ('limit' in c)) delete c.limit;
     return sortObject(c);
 };
 
@@ -380,12 +379,72 @@ export const addPendingWrites = async (builder, writeId, result) => {
     result = result && cloneDeep(result);
     await awaitStore();
 
+    const { projectUrl } = builder;
+    const pendingSnapshot = cloneDeep(result);
+    const { editions, linearWrite, pathChanges } = await syncCache(builder, result);
+
+    const isStaticWrite = !linearWrite.some(({ value, type }) => {
+        if (
+            [
+                'updateOne',
+                'updateMany',
+                'mergeOne',
+                'mergeMany'
+            ].includes(type)
+        ) {
+            const operators = Object.keys(value);
+            return ['$inc', '$min', '$max', '$mul', '$pop', '$pull', '$push', '$rename'].includes(operators);
+        }
+    });
+    const pureBuilder = {};
+
+    ['path', 'dbUrl', 'dbName', 'find', 'extraHeaders', 'maxRetries'].forEach(v => {
+        if (builder[v] !== undefined) pureBuilder[v] = builder[v];
+    });
+    pureBuilder.find = serializeToBase64({ _: pureBuilder.find });
+    pendingSnapshot.value = serializeToBase64({ _: pendingSnapshot.value });
+
+    let wasShifted;
+
+    if (isStaticWrite) {
+        // find previously matching pending write
+        const entries = Object.entries(CacheStore.PendingWrites[projectUrl] || {});
+
+        for (const [writeId, obj] of entries) {
+            if (!Scoped.OutgoingWrites[writeId]) {
+                if (
+                    niceGuard(
+                        { builder: obj.builder, snapshot: obj.snapshot },
+                        { builder: pureBuilder, snapshot: pendingSnapshot }
+                    )
+                ) {
+                    // shift it to the back
+                    obj.addedOn = Date.now();
+                    wasShifted = true;
+                    break;
+                }
+            }
+        }
+    }
+
+    if (!wasShifted)
+        poke(CacheStore.PendingWrites, [projectUrl, writeId], cloneDeep({
+            builder: pureBuilder,
+            snapshot: pendingSnapshot,
+            editions,
+            addedOn: Date.now()
+        }));
+
+    updateCacheStore(['DatabaseStore', 'PendingWrites', 'DatabaseStats']);
+    notifyDatabaseNodeChanges(builder, [...pathChanges]);
+};
+
+const syncCache = async (builder, result) => {
     const { io } = Scoped.ReleaseCacheData;
     const { projectUrl, dbUrl, dbName } = builder;
-    const editions = [];
     const duplicateSets = {};
+    const editions = [];
     const pathChanges = new Set([]);
-    const pendingSnapshot = cloneDeep(result);
 
     const linearWrite =
         result.type === 'batchWrite' ?
@@ -394,6 +453,8 @@ export const addPendingWrites = async (builder, writeId, result) => {
             )
             : [{ ...result, find: builder.find, path: builder.path }];
 
+    const copiedWrite = cloneDeep(linearWrite);
+
     await Promise.all(linearWrite.map(async ({ value: writeObj, find, type, path }) => {
         WriteValidator[type]({ find, value: writeObj });
         validateCollectionName(path);
@@ -421,18 +482,18 @@ export const addPendingWrites = async (builder, writeId, result) => {
             await Promise.all(colListing.map(async ([access_id]) =>
                 useFS(builder, access_id, 'database')(async fs => {
                     const data = await fs.find(LIMITER_DATA(path), access_id, ['value'])
-                        .then(r => deserializeBSON(r.value, true));
+                        .then(r => DatastoreParser.decode(r.value));
 
                     await MutateDataInstance([access_id, data], path =>
                         pathFinder[path] || (
                             pathFinder[path] = fs.list(LIMITER_DATA(path), ['value'])
-                                .then(v => v.map(d => deserializeBSON(d[1].value, true).data).flat())
+                                .then(v => v.map(d => DatastoreParser.decode(d[1].value).data).flat())
                                 .catch(() => [])
                         )
                     );
                     await fs.set(LIMITER_DATA(path), access_id, {
                         touched: Date.now(),
-                        value: serializeToBase64(data),
+                        value: DatastoreParser.encode(data),
                         size: data.size
                     });
                 })
@@ -639,132 +700,88 @@ export const addPendingWrites = async (builder, writeId, result) => {
         };
     }));
 
-    const isStaticWrite = !linearWrite.some(({ value, type }) => {
-        if (
-            [
-                'updateOne',
-                'updateMany',
-                'mergeOne',
-                'mergeMany'
-            ].includes(type)
-        ) {
-            const operators = Object.keys(value);
-            return ['$inc', '$min', '$max', '$mul', '$pop', '$pull', '$push', '$rename'].includes(operators);
-        }
-    });
-    const pureBuilder = {};
-
-    ['path', 'dbUrl', 'dbName', 'find', 'extraHeaders'].forEach(v => {
-        if (builder[v] !== undefined) pureBuilder[v] = builder[v];
-    });
-    pureBuilder.find = serializeToBase64({ _: pureBuilder.find });
-    pendingSnapshot.value = serializeToBase64({ _: pendingSnapshot.value });
-
-    let wasShifted;
-
-    if (isStaticWrite) {
-        // find previously matching pending write
-        const entries = Object.entries(CacheStore.PendingWrites[projectUrl] || {});
+    return {
+        editions,
+        pathChanges: [...pathChanges],
+        linearWrite: copiedWrite
+    };
+}
 
-        for (const [writeId, obj] of entries) {
-            if (!Scoped.OutgoingWrites[writeId]) {
-                if (
-                    niceGuard(
-                        { builder: obj.builder, snapshot: obj.snapshot },
-                        { builder: pureBuilder, snapshot: pendingSnapshot }
-                    )
-                ) {
-                    // shift it to the front
-                    obj.addedOn = Date.now();
-                    wasShifted = true;
-                    break;
-                }
-            }
-        }
-    }
+export const removePendingWrite = async (builder, writeId, revert) => {
+    await awaitStore();
+    const { projectUrl } = builder;
+    const pendingData = grab(CacheStore.PendingWrites, [projectUrl, writeId]);
+    if (!pendingData) return;
 
-    if (!wasShifted)
-        poke(CacheStore.PendingWrites, [projectUrl, writeId], cloneDeep({
-            builder: pureBuilder,
-            snapshot: pendingSnapshot,
-            editions,
-            addedOn: Date.now()
-        }));
+    const pathChanges = revert ? revertChanges(builder, pendingData.editions) : [];
 
-    updateCacheStore(['DatabaseStore', 'PendingWrites', 'DatabaseStats']);
+    unpoke(CacheStore.PendingWrites, [projectUrl, writeId]);
+    updateCacheStore(['PendingWrites', 'DatabaseStore', 'DatabaseStats']);
     notifyDatabaseNodeChanges(builder, [...pathChanges]);
 };
 
-export const removePendingWrite = async (builder, writeId, revert) => {
-    await awaitStore();
-    const { projectUrl, dbUrl, dbName } = builder;
-    const pendingData = grab(CacheStore.PendingWrites, [projectUrl, writeId]);
+const revertChanges = async (builder, pendingData) => {
     const { io } = Scoped.ReleaseCacheData;
-
-    if (!pendingData) return;
+    const { projectUrl, dbUrl, dbName } = builder;
     const pathChanges = new Set([]);
 
-    if (revert) {
-        await Promise.all(pendingData.editions.map(async ([access_id, [b4Doc, afDoc], path]) => {
-            if (io) {
-                RevertMutation(grab(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'instance', access_id]));
-            } else {
-                await useFS(builder, access_id, 'database')(async fs => {
-                    const colObj = await fs.find(LIMITER_DATA(path), access_id, ['value'])
-                        .then(v => deserializeBSON(v.value, true))
-                        .catch(() => null);
-                    if (!colObj) return;
-                    RevertMutation(colObj);
-                    await fs.set(LIMITER_DATA(path), access_id, {
-                        value: serializeToBase64(colObj),
-                        touched: Date.now(),
-                        size: colObj.size
-                    });
+    await Promise.all(pendingData.map(async ([access_id, [b4Doc, afDoc], path]) => {
+        if (io) {
+            RevertMutation(grab(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'instance', access_id]));
+        } else {
+            await useFS(builder, access_id, 'database')(async fs => {
+                const colObj = await fs.find(LIMITER_DATA(path), access_id, ['value'])
+                    .then(v => DatastoreParser.decode(v.value))
+                    .catch(() => null);
+                if (!colObj) return;
+                RevertMutation(colObj);
+                await fs.set(LIMITER_DATA(path), access_id, {
+                    value: DatastoreParser.encode(colObj),
+                    touched: Date.now(),
+                    size: colObj.size
                 });
-            }
+            });
+        }
 
-            function RevertMutation(colObj) {
-                const colList = colObj?.data;
+        function RevertMutation(colObj) {
+            const colList = colObj?.data;
 
-                const updateSize = (b4, af) => {
-                    const offset = docSize(af) - docSize(b4);
-                    colObj.size += offset;
-                    incrementDatabaseSize(builder, path, offset);
-                }
+            const updateSize = (b4, af) => {
+                const offset = docSize(af) - docSize(b4);
+                colObj.size += offset;
+                incrementDatabaseSize(builder, path, offset);
+            }
 
-                if (colList) {
-                    if (afDoc) {
-                        const editedIndex = colList.findIndex(e => CompareBson.equal(e._id, afDoc._id));
-                        if (editedIndex !== -1) {
-                            if (
-                                serializeToBase64(afDoc) === serializeToBase64(colList[editedIndex])
-                            ) {
-                                if (b4Doc) {
-                                    colList[editedIndex] = b4Doc;
-                                    updateSize(afDoc, b4Doc);
-                                } else {
-                                    colList.splice(editedIndex, 1);
-                                    updateSize(afDoc, undefined);
-                                }
+            if (colList) {
+                if (afDoc) {
+                    const editedIndex = colList.findIndex(e => CompareBson.equal(e._id, afDoc._id));
+                    if (editedIndex !== -1) {
+                        if (
+                            serializeToBase64(afDoc) === serializeToBase64(colList[editedIndex])
+                        ) {
+                            if (b4Doc) {
+                                colList[editedIndex] = b4Doc;
+                                updateSize(afDoc, b4Doc);
+                            } else {
+                                colList.splice(editedIndex, 1);
+                                updateSize(afDoc, undefined);
                             }
                         }
-                    } else if (
-                        b4Doc &&
-                        colList.findIndex(e => CompareBson.equal(e._id, b4Doc._id)) === -1
-                    ) {
-                        colList.push(b4Doc);
-                        updateSize(undefined, b4Doc);
                     }
+                } else if (
+                    b4Doc &&
+                    colList.findIndex(e => CompareBson.equal(e._id, b4Doc._id)) === -1
+                ) {
+                    colList.push(b4Doc);
+                    updateSize(undefined, b4Doc);
                 }
-                pathChanges.add(path);
             }
-        }));
-    }
+            pathChanges.add(path);
+        }
+    }));
 
-    unpoke(CacheStore.PendingWrites, [projectUrl, writeId]);
-    updateCacheStore(['PendingWrites', 'DatabaseStore', 'DatabaseStats']);
-    notifyDatabaseNodeChanges(builder, [...pathChanges]);
-};
+    return [...pendingData];
+}
 
 const notifyDatabaseNodeChanges = (builder, changedCollections = []) => {
     const { projectUrl, dbName, dbUrl } = builder;

package/src/products/database/bson.js

@@ -1,3 +1,4 @@
+import cloneDeep from "lodash/cloneDeep";
 import { deserialize, serialize } from "../../vendor/bson";
 import { Buffer } from "buffer";
 
@@ -13,4 +14,45 @@ export const deserializeBSON = (data, cast) => {
     });
 };
 
-export const serializeToBase64 = doc => Buffer.from(serialize(doc)).toString('base64');
+export const serializeToBase64 = doc => Buffer.from(serialize(doc)).toString('base64');
+
+export const DatastoreParser = {
+    encode: (obj) => {
+        obj = cloneDeep(obj);
+        const { command, config } = obj;
+
+        const serializeQuery = (e) =>
+            ['find', 'findOne'].forEach(n => {
+                if (e?.[n]) e[n] = serializeToBase64({ _: e[n] });
+            });
+
+        if (command) serializeQuery(command);
+        if (config) {
+            if (config.extraction)
+                (Array.isArray(config.extraction) ? config.extraction : [config.extraction]).forEach(e => {
+                    serializeQuery(e);
+                });
+        }
+        if (obj.data) obj.data = serializeToBase64({ _: obj.data });
+        return obj;
+    },
+    decode: (obj, cast = true) => {
+        obj = cloneDeep(obj);
+        const { command, config } = obj;
+
+        const serializeQuery = (e) =>
+            ['find', 'findOne'].forEach(n => {
+                if (e?.[n]) e[n] = deserializeBSON(e[n], cast)._;
+            });
+
+        if (command) serializeQuery(command);
+        if (config) {
+            if (config.extraction)
+                (Array.isArray(config.extraction) ? config.extraction : [config.extraction]).forEach(e => {
+                    serializeQuery(e);
+                });
+        }
+        if (obj.data) obj.data = deserializeBSON(obj.data, cast)._;
+        return obj;
+    }
+};

package/src/products/database/index.js

@@ -395,6 +395,7 @@ const countCollection = async (builder, config) => {
             const b4Data = await getCountQuery(builder, accessId).catch(() => null);
 
             if (e?.simpleError) {
+                e.simpleError.ack = true;
                 finalize(undefined, e.simpleError);
             } else if (!disableCache && Validator.NUMBER(b4Data)) {
                 finalize(b4Data);
@@ -552,6 +553,7 @@ const findObject = async (builder, config) => {
                 (thisRecord = [await getRecordData()])[0];
 
             if (e?.simpleError) {
+                e.simpleError.ack = true;
                 finalize(undefined, e?.simpleError);
             } else if (
                 (retrieval === RETRIEVAL.CACHE_NO_AWAIT && !(await getThisRecord())) ||
@@ -689,7 +691,8 @@ const commitData = async (builder, value, type, config) => {
         } catch (e) {
             if (e?.simpleError) {
                 console.error(`${type} error (${path}), ${e.simpleError?.message}`);
-                finalize(undefined, e?.simpleError, { removeCache: true, revertCache: true });
+                e.simpleError.ack = true;
+                finalize(undefined, e.simpleError, { removeCache: true, revertCache: true });
             } else if (delivery === DELIVERY.NO_CACHE_NO_AWAIT) {
                 finalize(undefined, simplifyCaughtError(e).simpleError);
             } else if (retries > maxRetries) {
@@ -738,9 +741,9 @@ export const trySendPendingWrite = (projectUrl) => {
 
                 delete CacheStore.PendingWrites[projectUrl][writeId];
                 ++resolveCounts;
-            } catch (_) {
+            } catch (err) {
                 const { maxRetries } = builder;
-                if (!maxRetries || attempts >= maxRetries) {
+                if (err?.ack || !maxRetries || attempts >= maxRetries) {
                     delete CacheStore.PendingWrites[projectUrl][writeId];
                     ++resolveCounts;
                 } else if (CacheStore.PendingWrites[projectUrl]?.[writeId]) {
@@ -749,8 +752,7 @@ export const trySendPendingWrite = (projectUrl) => {
             }
         }
         resolve();
-        if (projectUrl in Scoped.dispatchingWritesPromise)
-            delete Scoped.dispatchingWritesPromise[projectUrl];
+        Scoped.dispatchingWritesPromise[projectUrl] = undefined;
         updateCacheStore(['PendingWrites']);
 
         if (

package/src/products/database/types.js

@@ -1,4 +1,4 @@
-import { guardObject, GuardSignal, niceGuard } from "guard-object";
+import { niceGuard } from "guard-object";
 
 export const TIMESTAMP = { $timestamp: "now" };
 export const TIMESTAMP_OFFSET = (v) => ({ $timestamp_offset: v });
@@ -12,17 +12,13 @@ export const GEO_JSON = (lat, lng) => ({
     coordinates: [lng, lat],
 });
 
-export const FIND_GEO_JSON = (location, offSetMeters, centerMeters) => {
-    guardObject(GuardSignal.COORDINATE.LAT_LNG_INT).validate(location);
-
-    return {
-        $nearSphere: {
-            $geometry: {
-                type: "Point",
-                coordinates: location.slice(0).reverse()
-            },
-            $minDistance: centerMeters || 0,
-            $maxDistance: offSetMeters
-        }
-    };
-};
+export const FIND_GEO_JSON = (location, offSetMeters, centerMeters) => ({
+    $nearSphere: {
+        $geometry: {
+            type: "Point",
+            coordinates: location.slice(0).reverse()
+        },
+        $minDistance: centerMeters || 0,
+        $maxDistance: offSetMeters
+    }
+});

package/src/products/http_callable/accessor.js

@@ -40,6 +40,7 @@ export const getFetchResources = async (projectUrl, access_id) => {
     if (io) {
         const record = CacheStore.FetchedStore[projectUrl]?.[access_id];
         if (record) record.touched = Date.now();
+        updateCacheStore(['FetchedStore']);
         return record && cloneDeep(record?.data);
     }
 

package/src/products/http_callable/index.js

@@ -11,9 +11,9 @@ import { serialize } from "entity-serializer";
 import { getFetchResources, insertFetchResources } from "./accessor";
 
 const buildFetchData = (data, extras) => {
-    const { ok, type, status, statusText, redirected, url, headers, size, base64 } = data;
+    const { ok, type, status, statusText, redirected, url, headers, size, buffer } = data;
 
-    const response = new Response(Buffer.from(base64, 'base64'), {
+    const response = new Response(buffer, {
         headers: new Headers(headers),
         status,
         statusText,
@@ -146,7 +146,7 @@ export const mfetch = async (input = '', init, config) => {
                 credentials: 'omit',
                 ...init,
                 ...uglified ? { body: reqBuilder } : encodeBody ? { body: serialize(body) } : {},
-                // cache: 'no-cache',
+                cache: 'no-cache',
                 headers: {
                     ...extraHeaders,
                     ...rawHeader,
@@ -165,12 +165,12 @@ export const mfetch = async (input = '', init, config) => {
 
             if (!isLink && simple) throw { simpleError: JSON.parse(simple) };
 
-            const base64 = uglified ?
-                Buffer.from(await deserializeE2E(await f.arrayBuffer(), serverE2E_PublicKey, privateKey)).toString('base64') :
-                Buffer.from(await f.arrayBuffer()).toString('base64');
+            const buffer = uglified ?
+                Buffer.from(await deserializeE2E(await f.arrayBuffer(), serverE2E_PublicKey, privateKey)) :
+                Buffer.from(await f.arrayBuffer());
 
             const resObj = {
-                base64,
+                buffer,
                 type,
                 status,
                 statusText,