react-native-mosquito-transport 0.0.21 → 0.0.23

package/src/helpers/utils.js

@@ -1,21 +1,32 @@
-import AsyncStorage from "@react-native-async-storage/async-storage";
 import { ServerReachableListener, StoreReadyListener } from "./listeners";
-import { CACHE_PROTOCOL, CACHE_STORAGE_PATH, DEFAULT_CACHE_PASSWORD, LOCAL_STORAGE_PATH } from "./values";
 import { CacheStore, Scoped } from "./variables";
-import { decryptString, encryptString, niceTry, serializeE2E } from "./peripherals";
+import { serializeE2E } from "./peripherals";
 import { deserializeBSON, serializeToBase64 } from "../products/database/bson";
 import { trySendPendingWrite } from "../products/database";
 import { deserialize } from "entity-serializer";
+import { openDB, SQLITE_COMMANDS, SQLITE_PATH } from "./sqlite_manager";
+import { purgeRedundantRecords } from "./purger";
+import { getStoreID, handleBigData, parseBigData } from "./fs_manager";
 
-export const updateCacheStore = (timer = 300) => {
-    const { cachePassword = DEFAULT_CACHE_PASSWORD, cacheProtocol = CACHE_PROTOCOL.ASYNC_STORAGE, io, promoteCache } = Scoped.ReleaseCacheData;
+const { FILE_NAME, TABLE_NAME } = SQLITE_PATH;
 
-    clearTimeout(Scoped.cacheStorageReducer);
-    Scoped.cacheStorageReducer = setTimeout(() => {
-        const { AuthStore, EmulatedAuth, PendingAuthPurge, DatabaseStore, PendingWrites, ...restStore } = CacheStore;
+const CacheKeys = Object.keys(CacheStore);
+
+export const updateCacheStore = (timer = 300, node) => {
+    const { io, promoteCache } = Scoped.ReleaseCacheData;
+
+    const doUpdate = async () => {
+        const {
+            AuthStore,
+            EmulatedAuth,
+            PendingAuthPurge,
+            DatabaseStore,
+            PendingWrites,
+            ...restStore
+        } = CacheStore;
 
-        const txt = encryptString(
-            JSON.stringify({
+        if (io) {
+            const txt = JSON.stringify({
                 AuthStore,
                 EmulatedAuth,
                 PendingAuthPurge,
@@ -24,51 +35,75 @@ export const updateCacheStore = (timer = 300) => {
                     PendingWrites: serializeToBase64(PendingWrites)
                 } : {},
                 ...promoteCache ? restStore : {}
-            }),
-            cachePassword,
-            cachePassword
-        );
+            });
 
-        if (io) {
-            io.output(txt);
-        } else if (cacheProtocol === CACHE_PROTOCOL.ASYNC_STORAGE) {
-            AsyncStorage.setItem(CACHE_STORAGE_PATH, txt);
-        } else if (cacheProtocol === CACHE_PROTOCOL.SQLITE) {
-            console.error('unsupported protocol "sqlite"');
+            io.output(txt, node);
         } else {
-            const fs = require('react-native-fs');
-            fs.writeFile(LOCAL_STORAGE_PATH(), txt, 'utf8');
+            // use sqlite
+            const exclusion = ['DatabaseStore', 'DatabaseCountResult', 'FetchedStore'];
+            const updationKey = (node ? Array.isArray(node) ? node : [node] : CacheKeys).filter(v => !exclusion.includes(v));
+
+            if (!updationKey.length) return;
+            const sqlite = await openDB(FILE_NAME);
+            await Promise.all(
+                updationKey
+                    .map(v => [v, v === 'PendingWrites' ? serializeToBase64(CacheStore[v]) : CacheStore[v]])
+                    .map(async ([ref, value]) => {
+                        const blobData = await handleBigData(getStoreID(FILE_NAME, TABLE_NAME, ref), value);
+                        return sqlite.executeSql(SQLITE_COMMANDS.MERGE(TABLE_NAME, ['ref', 'value']), [ref, blobData]);
+                    })
+            ).catch(err => {
+                console.error('updateCacheStore err:', err);
+            }).finally(() => {
+                sqlite.close();
+            });
         }
-    }, timer);
+    };
+
+    clearTimeout(Scoped.cacheStorageReducer);
+    if (timer) {
+        Scoped.cacheStorageReducer = setTimeout(doUpdate, timer);
+    } else doUpdate();
 };
 
 export const releaseCacheStore = async (builder) => {
-    const { cachePassword = DEFAULT_CACHE_PASSWORD, cacheProtocol = CACHE_PROTOCOL.ASYNC_STORAGE, io } = builder;
+    const { io } = builder;
 
-    let txt;
+    let data = {};
 
-    if (io) {
-        txt = await io.input();
-    } else if (cacheProtocol === CACHE_PROTOCOL.ASYNC_STORAGE) {
-        txt = await niceTry(() => AsyncStorage.getItem(CACHE_STORAGE_PATH));
-    } else {
-        const fs = require('react-native-fs');
-        txt = await niceTry(() => fs.readFile(LOCAL_STORAGE_PATH(), 'utf8'));
+    try {
+        if (io) {
+            data = JSON.parse((await io.input()) || '{}');
+        } else {
+            const sqlite = await openDB(FILE_NAME);
+            await sqlite.executeSql(`CREATE TABLE IF NOT EXISTS ${TABLE_NAME} ( ref TEXT PRIMARY KEY, value BLOB )`).catch(() => null);
+            try {
+                const [query] = await sqlite.executeSql(`SELECT * FROM ${TABLE_NAME}`);
+                data = Object.fromEntries(
+                    await Promise.all(query.rows.raw().map(async v =>
+                        [v.ref, await parseBigData(v.value)]
+                    ))
+                );
+            } catch (error) {
+                console.error('initializeCache sqlite data release err:', error);
+            } finally {
+                sqlite.close();
+            }
+        }
+        await purgeRedundantRecords(data, builder);
+    } catch (e) {
+        console.error('initializeCache data err:', e);
     }
 
-    const j = JSON.parse(decryptString(txt || '', cachePassword, cachePassword) || '{}');
-    const projectList = new Set([]);
-
-    Object.entries(j).forEach(([k, v]) => {
+    Object.entries(data).forEach(([k, v]) => {
         if (['DatabaseStore', 'PendingWrites'].includes(k)) {
             CacheStore[k] = deserializeBSON(v);
         } else CacheStore[k] = v;
-        projectList.add(k);
     });
     Object.entries(CacheStore.AuthStore).forEach(([key, value]) => {
         Scoped.AuthJWTToken[key] = value?.token;
     });
-    projectList.forEach(projectUrl => {
+    Object.keys(CacheStore.PendingWrites).forEach(projectUrl => {
         if (Scoped.IS_CONNECTED[projectUrl])
             trySendPendingWrite(projectUrl);
     });
@@ -117,18 +152,17 @@ export const getReachableServer = (projectUrl) => new Promise(resolve => {
     }, true);
 });
 
-export const buildFetchInterface = async ({ body, accessKey, authToken, method, uglify, serverE2E_PublicKey, extraHeaders }) => {
+export const buildFetchInterface = async ({ body, authToken, method, uglify, serverE2E_PublicKey, extraHeaders }) => {
     if (!uglify) body = JSON.stringify({ ...body });
     const [plate, keyPair] = uglify ? await serializeE2E(body, authToken, serverE2E_PublicKey) : [undefined, []];
 
     return [{
         body: uglify ? plate : body,
-        cache: 'no-cache',
+        // cache: 'no-cache',
         headers: {
+            ...extraHeaders,
             'Content-type': uglify ? 'request/buffer' : 'application/json',
-            'Authorization': accessKey,
-            ...(authToken && !uglify) ? { 'Mosquito-Token': authToken } : {},
-            ...extraHeaders
+            ...(authToken && !uglify) ? { 'Mosquito-Token': authToken } : {}
         },
         method: method || 'POST'
     }, keyPair];

package/src/helpers/values.js

@@ -1,24 +1,10 @@
-import { Platform } from 'react-native';
-import { encodeBinary } from './peripherals';
-
-export const CACHE_STORAGE_PATH = encodeBinary('MOSQUITO_TRANSPORT_FREEZER'),
-    DEFAULT_CACHE_PASSWORD = encodeBinary('MOSQUITO_TRANSPORT_CACHE_PASSWORD'),
-    LOCAL_STORAGE_PATH = () => {
-        const fs = require('react-native-fs');
-        return `${Platform.OS === 'android' ? fs.ExternalCachesDirectoryPath : fs.CachesDirectoryPath}/${encodeBinary('MOSQUITO_TRANSPORT_STORAGE')}`;
-    };
-
-export const CACHE_PROTOCOL = {
-    ASYNC_STORAGE: 'async-storage',
-    REACT_NATIVE_FS: 'reat-native-fs',
-    SQLITE: 'sqlite' // TODO:
-};
 
 export const RETRIEVAL = {
     STICKY: 'sticky',
     STICKY_NO_AWAIT: 'sticky-no-await',
     STICKY_RELOAD: 'sticky-reload',
     DEFAULT: 'default',
+    CACHE_AWAIT: 'cache-await',
     CACHE_NO_AWAIT: 'cache-no-await',
     NO_CACHE_NO_AWAIT: 'no-cache-no-await',
     NO_CACHE_AWAIT: 'no-cache-await'
@@ -26,11 +12,10 @@ export const RETRIEVAL = {
 
 export const DELIVERY = {
     DEFAULT: 'default',
-    NO_CACHE: 'no-cache',
-    NO_AWAIT: 'no-await',
-    NO_AWAIT_NO_CACHE: 'no-await-no-cache',
-    AWAIT_NO_CACHE: 'await-no-cache',
-    CACHE_NO_AWAIT: 'cache-no-await'
+    CACHE_AWAIT: 'cache-await',
+    CACHE_NO_AWAIT: 'cache-no-await',
+    NO_CACHE_NO_AWAIT: 'no-cache-no-await',
+    NO_CACHE_AWAIT: 'no-cache-await'
 };
 
 export const AUTH_PROVIDER_ID = {

package/src/helpers/variables.js

@@ -14,26 +14,52 @@ export const Scoped = {
     LastTokenRefreshRef: {},
     StorageProcessID: 0,
     InitiatedForcedToken: {},
-    PendingFetchCollective: {
-        pendingProcess: {},
-        pendingResolution: {}
-    },
-    PendingDbReadCollective: {
-        pendingProcess: {},
-        pendingResolution: {}
-    },
+    PendingFetchCollective: {},
+    PendingDbReadCollective: {},
     ActiveDatabaseListeners: {},
     OutgoingWrites: {},
     /**
      * @type {Promise<any> | undefined}
      */
-    dispatchingWritesPromise: undefined
+    dispatchingWritesPromise: undefined,
+    linearSqliteProcess: {
+        database: {},
+        dbQueryCount: {},
+        httpFetch: {}
+    },
+    initedSqliteInstances: {
+        httpFetch: {},
+        dbQueryCount: {},
+        database: {}
+    }
+};
+
+export const SqliteCollective = {
+    openedDb: {},
+    openedDbProcess: {},
+    closeDbPromises: {},
+    openedDbReducerTimer: {}
 };
 
 export const CacheStore = {
     DatabaseStore: {},
     DatabaseCountResult: {},
-    DatabaseStats: {},
+    DatabaseStats: {
+        /**
+         * @type {{[projectUrl: string]: {[dbUrl: string]: {[dbName: string]: {[path: string]: number}}}}}
+         */
+        database: {},
+        /**
+         * @type {{[projectUrl: string]: {[dbUrl: string]: {[dbName: string]: {[path: string]: boolean}}}}}
+         */
+        counters: {},
+        /**
+         * @type {{[projectUrl: string]: number}}
+         */
+        fetchers: {},
+        _db_size: 0,
+        _fetcher_size: 0
+    },
     AuthStore: {},
     PendingAuthPurge: {},
     EmulatedAuth: {},

package/src/index.d.ts

@@ -2,10 +2,16 @@ interface RNMTConfig {
     dbName?: string;
     dbUrl?: string;
     projectUrl: string;
+    /**
+     * disable caching for database and fetchHttp in this instance.
+     * 
+     * defaults to `false` but `true` in scenerio where an invoked fetchHttp does have a request body and disableCache is undefined
+     */
     disableCache?: boolean;
-    accessKey: string;
     /**
-     * maximum numbers of attempts to retry sending a request
+     * maximum numbers of attempts to retry sending a failed request
+     * 
+     * @default 3
      */
     maxRetries?: number;
     /**
@@ -37,6 +43,7 @@ interface GetDatabase {
 interface mtimestamp { $timestamp: 'now' }
 
 export const TIMESTAMP: mtimestamp;
+export function TIMESTAMP_OFFSET(offset: number): { $timestamp_offset: number };
 export function DOCUMENT_EXTRACTION(path: string): { $dynamicValue: string };
 
 type longitude = number;
@@ -64,6 +71,7 @@ export const AUTH_PROVIDER_ID: auth_provider_id;
  * useful for avoiding encrypting data and extra overhead
  */
 export class DoNotEncrypt {
+    constructor(value: any);
     value: any;
 }
 
@@ -84,40 +92,45 @@ type auth_provider_id_values = auth_provider_id['GOOGLE'] |
     auth_provider_id['APPLE'];
 
 interface ReleaseCacheOption_IO {
+}
+
+interface ReleaseCacheOption {
     /**
-     * This password will be used to encrypt data stored locally
+     * This key will be used to encrypt sqlite data stored locally
+     */
+    sqliteKey?: string;
+    /**
+     * sqlite is used as the default caching mechanism
+     * 
+     * providing `io` means you want to override this behaviour and handle caching yourself in the system's memory
      */
-    cachePassword?: string;
     io: {
         /**
-         * feeds mosquito-transport data
+         * you should handle retrieval of mosquito-transport data here by returning it
          */
-        input: () => string;
+        input: () => string | Promise<string>;
         /**
-         * emits mosquito-transport internal data
+         * you should handle storage of mosquito-transport data here by caching the emitted string
          */
         output: (data: string) => void;
-    };
-}
-
-interface ReleaseCacheOption {
-    /**
-     * This password will be used to encrypt data stored locally
-     */
-    cachePassword?: string;
+    },
     /**
-     * select the mechanism for storing data locally
-     * - async-storage: uses [@react-native-async-storage/async-storage](https://github.com/react-native-async-storage/async-storage) for storing data, make sure you have this package installed before setting this as the value
-     * - reat-native-fs: uses [reat-native-fs](https://github.com/itinance/react-native-fs) for storing data, make sure you have this package installed before setting this as the value
+     * the maximum size allocated to local database
+     * 
+     * when local database size exceed this value, redundant database documents will be purge retaining only recently used ones
+     * 
+     * @default 10485760 - 'which is 10 megabytes'
      */
-    cacheProtocol?: 'async-storage' | 'reat-native-fs';
+    maxLocalDatabaseSize?: number;
     /**
-     * the maximum database size
+     * the maximum size allocated to local database for storing {@link RNMT.fetchHttp}
+     * 
+     * @default 10485760 - 'which is 10 megabytes'
      */
-    heapMemory?: number;
+    maxLocalFetchHttpSize?: number;
     /**
      * true to cache database and {@link RNMT.fetchHttp} data.
-     * the value only applies to in-memory storage protocol such as `io`, `async-storage` and `reat-native-fs`
+     * the value only applies to in-memory storage protocol such as `io`
      * 
      * @default false
      */
@@ -146,16 +159,28 @@ interface BatchWriteConfig extends WriteConfig {
     stepping?: boolean;
 }
 
+interface CountConfig {
+    disableAuth?: boolean;
+}
+
+interface FetchHttpResponse extends Response {
+    /**
+     * true if this response was from local cache
+     */
+    fromCache?: boolean | undefined;
+}
+
 export default class RNMT {
     constructor(config: RNMTConfig);
-    static initializeCache(option?: ReleaseCacheOption | ReleaseCacheOption_IO): void;
+    static initializeCache(option?: ReleaseCacheOption): void;
     getDatabase(dbName?: string, dbUrl?: string): GetDatabase;
     collection(path: string): RNMTCollection;
     auth(): RNMTAuth;
     storage(): RNMTStorage;
-    fetchHttp(endpoint: string, init?: RequestInit, config?: FetchHttpConfig): Promise<Response>;
+    fetchHttp(endpoint: string, init?: RequestInit, config?: FetchHttpConfig): Promise<FetchHttpResponse>;
     listenReachableServer(callback: (reachable: boolean) => void): () => void;
     getSocket(options: { disableAuth?: boolean; authHandshake?: Object }): RNMTSocket;
+    onConnect: () => CollectionIO;
     batchWrite(map: BatchWriteValue[], config?: BatchWriteConfig): Promise<DocumentWriteResult[] | undefined>;
 }
 
@@ -163,7 +188,7 @@ interface RNMTCollection {
     find: (find?: DocumentFind) => ({
         get: (config?: GetConfig) => Promise<DocumentResult[]>;
         listen: (callback: (snapshot?: DocumentResult[]) => void, onError?: (error?: DocumentError) => void, config?: GetConfig) => void;
-        count: () => Promise<number>;
+        count: (config?: CountConfig) => Promise<number>;
         limit: (limit: number) => ({
             random: (config?: GetConfig) => Promise<DocumentResult[]>;
             get: (config?: GetConfig) => Promise<DocumentResult[]>;
@@ -199,25 +224,13 @@ interface RNMTCollection {
             listen: (callback: (snapshot?: DocumentResult[]) => void, onError?: (error?: DocumentError) => void, config?: GetConfig) => void;
         })
     });
-    count: () => Promise<number>;
+    count: (config?: CountConfig) => Promise<number>;
     get: (config?: GetConfig) => Promise<DocumentResult[]>;
     listen: (callback: (snapshot?: DocumentResult[]) => void, onError?: (error?: DocumentError) => void, config?: GetConfig) => void;
     findOne: (findOne?: DocumentFind) => ({
         get: (config?: GetConfig) => Promise<DocumentResult>;
         listen: (callback: (snapshot?: DocumentResult) => void, onError?: (error?: DocumentError) => void, config?: GetConfig) => void;
     });
-    onDisconnect: () => ({
-        setOne: (value: DocumentWriteValue) => () => void;
-        setMany: (value: DocumentWriteValue[]) => () => void;
-        updateOne: (find: DocumentFind, value: DocumentWriteValue) => () => void;
-        updateMany: (find: DocumentFind, value: DocumentWriteValue) => () => void;
-        mergeOne: (find: DocumentFind, value: DocumentWriteValue) => () => void;
-        mergeMany: (find: DocumentFind, value: DocumentWriteValue) => () => void;
-        replaceOne: (find: DocumentFind, value: DocumentWriteValue) => () => void;
-        putOne: (find: DocumentFind, value: DocumentWriteValue) => () => void;
-        deleteOne: (find: DocumentFind) => () => void;
-        deleteMany: (find?: DocumentFind) => () => void;
-    })
 
     setOne: (value: DocumentWriteValue, config?: WriteConfig) => Promise<DocumentWriteResult>;
 
@@ -240,8 +253,24 @@ interface RNMTCollection {
     deleteMany: (find?: DocumentFind, config?: WriteConfig) => Promise<DocumentWriteResult>;
 }
 
-interface DocumentResult {
-    _id: any
+interface CollectionTaskIO<T> {
+    batchWrite(map: BatchWriteValue[], config?: BatchWriteConfig): T;
+}
+
+interface StartStop {
+    start: () => () => void;
+}
+
+interface OnConnectChain extends StartStop {
+    onDisconnect: () => CollectionTaskIO<StartStop>;
+}
+
+interface CollectionIO extends CollectionTaskIO<OnConnectChain> {
+    onDisconnect: () => CollectionTaskIO<StartStop>;
+}
+
+interface DocumentResult extends Document {
+    _foreign_doc?: Document | Document[] | Document[][]
 }
 
 interface DocumentError extends ErrorResponse {
@@ -263,15 +292,21 @@ interface WriteConfig {
      */
     disableAuth?: boolean;
 
+    DEFAULT: 'default',
+    CACHE_AWAIT: 'cache-await',
+    CACHE_NO_AWAIT: 'cache-no-await',
+    NO_CACHE_NO_AWAIT: 'no-cache-no-await',
+    NO_CACHE_AWAIT: 'no-cache-await'
+
     /**
-     * This property defines how the write will be committed and sent
+     * This property determines how the write will be dispatch to the remote server
+     *  TODO:
      * 
-     * - default: 
-     * - no-cache: 
-     * - no-await: 
-     * - no-await-no-cache: 
-     * - await-no-cache: 
-     * - cache-no-await: 
+     * `default`: 
+     * `cache-await`: 
+     * `cache-no-await`: 
+     * `no-cache-no-await`: 
+     * `no-cache-await`: 
      */
     delivery?: Delievery;
 }
@@ -283,23 +318,28 @@ interface GetConfig {
     returnOnly?: string | string[];
     extraction?: GetConfigExtraction | GetConfigExtraction[];
     /**
-     * This property determines how requested data is being access and handled
+     * This property determines how requested data should be accessed and handled
      * 
-     * - default: will try getting fresh data from server if server is reachable, else we check if local data exists and return it, if no local data exist we await for client->server connectivity, then try getting the data, return it and cache it for future need. ⚠️ This respect disableCache value
+     *  `default`: will try getting fresh data from the server if the server is reachable then cache the result for future use. In scenerio where the server is unreachable, we check if local data exists and return it, if no local data exist we wait for reachable a server to complete the task.
+     * ⚠️ This respect `disableCache` value therefore if disableCache is true, no local cache read/write will be made for the request.
      * 
-     * - sticky: if local data exists for the specified query, such data is returned and no-ops afterwards. If no local data is found, we await for client->server connectivity and try to get fresh data from serve, return the data and cache it for future need. ⚠️ This does not respect disableCache value
+     *  `sticky`: if local data exists for the specified query, such data is returned and no-ops afterwards. If no local data is found, we await for client->server connectivity and try to get fresh data from serve, return the data and cache it for future need. ⚠️ This overrides `disableCache` value
      * 
-     * - sticky-no-await: if local data exists for the specified query, such data is returned and no-ops afterwards. If no local data is found, will try to get fresh data from serve, we throw an error if server is not reachable else if server returns requested data, we return such data and cache it for future need. ⚠️ This does not respect disableCache value
+     * `sticky-no-await`: if local data exists for the specified query, such data is returned and no operation is performed afterwards. If no local data is found, will try to get fresh data from serve, we throw an error if server is unreachable else if server returns requested data, we return such data and cache it for future need. ⚠️ This overrides `disableCache` value
      * 
-     * - sticky-reload: if local data exists for the specified query, such data is returned, then will try getting fresh data from the server and caching it for future need. If no local data is found, we await for client->server connectivity and try to get fresh data from serve, return the data and cache it for future need
+     * `sticky-reload`: if local data exists for the specified query, such data is returned, then will try getting fresh data from the server and caching it for future need. If no local data is found, we await for a reachable server and try to get fresh data from the serve, return the data and cache it for future need. ⚠️ This overrides `disableCache` value
      * 
-     * - cache-no-await: will try getting fresh data from server if server is reachable, else we check if local data exists and return it, if no local data exist we throw an error
+     * `cache-await`: will try getting fresh data from the server if the server is reachable then cache the result for future use. In scenerio where the server is unreachable, then we check if local data exists and return it, if no local data exist we wait for a reachable server to complete the task. ⚠️ This overrides `disableCache` value
      * 
-     * - no-cache-no-await: will try getting fresh data from server if server is reachable, else we throw an error
+     * `cache-no-await`: will try getting fresh data from the server if the server is reachable then cache the result for future use. In scenerio where the server is unreachable, then we check if local data exists and return it, if no local data exist an error is thrown instead. ⚠️ This overrides `disableCache` value
      * 
-     * - no-cache-await: will try getting fresh data from server when server is reachable and the result won't be cache
+     * `no-cache-no-await`: will try getting fresh data from the server and the result won't be cached. In scenerio where the server is unreachable, an error will be thrown. ⚠️ This overrides `disableCache` value
+     * 
+     * `no-cache-await`: will wait for a reachable server then try getting fresh data from that server and the result won't be cache. ⚠️ This overrides `disableCache` value
      * 
      * To learn and see more examples on this, Please visit https://brainbehindx.com/mosquito-transport/docs/reading_data/retrieval
+     * 
+     * @default 'default'
      */
     retrieval?: Retrieval;
     /**
@@ -324,7 +364,7 @@ interface GetConfig {
      * 
      * ```js
      * 
-     * const mserver = new RNMT({ projectUrl: 'http..', accessKey: '..'});
+     * const mserver = new RNMT({ projectUrl: 'http://..' });
      * 
      * // operation will be reduced to only one request
      * 
@@ -375,6 +415,7 @@ interface DocumentFind {
 }
 
 declare interface Document {
+    _id: any;
     [key: string]: any;
 }