react-native-mosquito-transport 0.0.32 → 0.0.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -8
- package/src/helpers/fs_manager.js +88 -39
- package/src/helpers/peripherals.js +3 -3
- package/src/helpers/purger.js +23 -41
- package/src/helpers/utils.js +42 -58
- package/src/helpers/variables.js +3 -6
- package/src/index.d.ts +1 -5
- package/src/index.js +1 -4
- package/src/products/auth/accessor.js +3 -3
- package/src/products/auth/index.js +8 -9
- package/src/products/database/accessor.js +70 -134
- package/src/products/database/index.js +1 -1
- package/src/products/http_callable/accessor.js +12 -35
- package/src/products/http_callable/index.js +1 -1
- package/src/helpers/sqlite_manager.js +0 -144
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "react-native-mosquito-transport",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.34",
|
|
4
4
|
"description": "React native javascript sdk for mosquito-transport (https://github.com/brainbehindx/mosquito-transport)",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -40,12 +40,8 @@
|
|
|
40
40
|
},
|
|
41
41
|
"peerDependencies": {
|
|
42
42
|
"react-native": "*",
|
|
43
|
-
"react-native-
|
|
44
|
-
"react-native-
|
|
45
|
-
"react-native-
|
|
46
|
-
"react-native-get-random-values": "*"
|
|
47
|
-
},
|
|
48
|
-
"devDependencies": {
|
|
49
|
-
"@types/react-native-sqlite-storage": "^6.0.5"
|
|
43
|
+
"react-native-file-access": "*",
|
|
44
|
+
"react-native-get-random-values": "*",
|
|
45
|
+
"react-native-sha256": "*"
|
|
50
46
|
}
|
|
51
47
|
}
|
|
@@ -1,27 +1,82 @@
|
|
|
1
|
+
import { Scoped } from "./variables";
|
|
1
2
|
import { Platform } from "react-native";
|
|
2
|
-
import {
|
|
3
|
-
import { deserialize, serialize } from 'entity-serializer';
|
|
4
|
-
import { writeFile, mkdir, MainBundlePath, readFile, unlink, DocumentDirectoryPath } from "react-native-fs";
|
|
3
|
+
import { Dirs, FileSystem } from "react-native-file-access";
|
|
5
4
|
|
|
6
|
-
const
|
|
5
|
+
const PARENT_FOLDER = `${Platform.OS === 'android' ? Dirs.DocumentDir.split('/').slice(0, -1).join('/') : Dirs.MainBundleDir}/mosquito_base`;
|
|
7
6
|
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
/**
|
|
8
|
+
* this method linearize read/write for individual access_id on the file system ensuring consistency across concurrent operations
|
|
9
|
+
*
|
|
10
|
+
* @param {any} builder
|
|
11
|
+
* @param {string} access_id
|
|
12
|
+
* @param {string} node
|
|
13
|
+
* @returns {(task: (system: { set: (table: string, primary_key: string, value: {}) => Promise<void>, delete: (table: string, primary_key: string) => Promise<void>, find: (table: string, primary_key: string, extractions: string[]) => Promise<{}>, list: (table: string, extractions: string[]) => Promise<[string, {}][]> }) => any) => Promise<any>}
|
|
14
|
+
*/
|
|
15
|
+
export const useFS = (builder, access_id, node) => async (task) => {
|
|
16
|
+
const { projectUrl, dbUrl, dbName } = builder;
|
|
17
|
+
const nodeId = typeof builder === 'string' ? `${builder}_${access_id}` : `${projectUrl}_${dbUrl}_${dbName}_${access_id}`;
|
|
10
18
|
|
|
11
|
-
const
|
|
19
|
+
const thatProcess = Scoped.linearSqliteProcess[node][nodeId];
|
|
12
20
|
|
|
13
|
-
const
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
}
|
|
21
|
+
const thisPromise = new Promise(async (resolve, reject) => {
|
|
22
|
+
try {
|
|
23
|
+
if (thatProcess !== undefined) await thatProcess;
|
|
24
|
+
} catch (_) { }
|
|
25
|
+
try {
|
|
26
|
+
resolve(await task(getSystem(builder)));
|
|
27
|
+
} catch (error) {
|
|
28
|
+
console.error('useFS err:', error, ' builder:', builder);
|
|
29
|
+
reject(error);
|
|
30
|
+
} finally {
|
|
31
|
+
if (Scoped.linearSqliteProcess[node][nodeId] === thisPromise)
|
|
32
|
+
delete Scoped.linearSqliteProcess[node][nodeId];
|
|
33
|
+
}
|
|
34
|
+
});
|
|
17
35
|
|
|
18
|
-
|
|
19
|
-
await
|
|
20
|
-
|
|
21
|
-
|
|
36
|
+
Scoped.linearSqliteProcess[node][nodeId] = thisPromise;
|
|
37
|
+
return (await thisPromise);
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export const getSystem = (builder) => {
|
|
41
|
+
const { projectUrl, dbUrl, dbName } = builder;
|
|
42
|
+
|
|
43
|
+
const DIR_PATH = joinPath(PARENT_FOLDER, purifyFilepath(typeof builder === 'string' ? builder : `${projectUrl}_${dbUrl}_${dbName}`));
|
|
44
|
+
const conjoin = (...args) => joinPath(DIR_PATH, ...args);
|
|
22
45
|
|
|
23
|
-
|
|
24
|
-
|
|
46
|
+
return {
|
|
47
|
+
set: async (table, primary_key, value) => {
|
|
48
|
+
const path = conjoin(table, primary_key);
|
|
49
|
+
await FileSystem.mkdir(path).catch(() => null);
|
|
50
|
+
await Promise.all(Object.entries(value).map(([k, v]) =>
|
|
51
|
+
FileSystem.writeFile(joinPath(path, k), JSON.stringify(v), 'utf8')
|
|
52
|
+
));
|
|
53
|
+
},
|
|
54
|
+
delete: (table, primary_key) => FileSystem.unlink(conjoin(table, primary_key)),
|
|
55
|
+
find: async (table, primary_key, extractions) => {
|
|
56
|
+
const path = conjoin(table, primary_key);
|
|
57
|
+
|
|
58
|
+
const value_map = await Promise.all(extractions.map(async node =>
|
|
59
|
+
[node, JSON.parse(await FileSystem.readFile(joinPath(path, node)), 'utf8')]
|
|
60
|
+
));
|
|
61
|
+
return Object.fromEntries(value_map);
|
|
62
|
+
},
|
|
63
|
+
list: async (table, extractions) => {
|
|
64
|
+
const names = await FileSystem.ls(conjoin(table));
|
|
65
|
+
const list_data = await Promise.all(names.map(async primary_key => {
|
|
66
|
+
const obj = await system.find(table, primary_key, extractions)
|
|
67
|
+
.catch(() => null);
|
|
68
|
+
if (!obj) return;
|
|
69
|
+
return [primary_key, obj];
|
|
70
|
+
}));
|
|
71
|
+
|
|
72
|
+
return list_data.filter(v => v);
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
export function purifyFilepath(filename) {
|
|
78
|
+
if (!filename || typeof filename !== 'string')
|
|
79
|
+
throw `invalid filename:${filename}`;
|
|
25
80
|
|
|
26
81
|
// Remove invalid characters for both iOS and Android
|
|
27
82
|
return filename
|
|
@@ -29,27 +84,21 @@ const purifyFilename = (filename) => {
|
|
|
29
84
|
.trim(); // Remove leading/trailing whitespace
|
|
30
85
|
}
|
|
31
86
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
await fsWrite(store_id, bufData);
|
|
42
|
-
return serialize([undefined, store_id]).toString('base64');
|
|
43
|
-
};
|
|
87
|
+
function joinPath(...args) {
|
|
88
|
+
return args.map((v, i) => {
|
|
89
|
+
if (i && v.startsWith('/'))
|
|
90
|
+
v = v.slice(1);
|
|
91
|
+
if (v.endsWith('/'))
|
|
92
|
+
v = v.slice(0, -1);
|
|
93
|
+
return v;
|
|
94
|
+
}).join('/');
|
|
95
|
+
}
|
|
44
96
|
|
|
45
|
-
export const
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
return deserialize(inline);
|
|
97
|
+
export const FS_PATH = {
|
|
98
|
+
FILE_NAME: 'MOSQUITO_TRANSPORT',
|
|
99
|
+
TABLE_NAME: 'MT_MAIN',
|
|
100
|
+
LIMITER_RESULT: path => `${purifyFilepath(encodeURIComponent(path))}_LR`,
|
|
101
|
+
LIMITER_DATA: path => `${purifyFilepath(encodeURIComponent(path))}_LD`,
|
|
102
|
+
DB_COUNT_QUERY: path => `${purifyFilepath(encodeURIComponent(path))}_QC`,
|
|
103
|
+
FETCH_RESOURCES: projectUrl => `FR_${purifyFilepath(encodeURIComponent(projectUrl))}`
|
|
55
104
|
};
|
|
@@ -4,6 +4,7 @@ import naclPkg from 'tweetnacl';
|
|
|
4
4
|
import getLodash from "lodash/get";
|
|
5
5
|
import { deserialize, serialize } from "entity-serializer";
|
|
6
6
|
import { sha256 } from 'react-native-sha256';
|
|
7
|
+
import { purifyFilepath } from "./fs_manager";
|
|
7
8
|
|
|
8
9
|
const { box, randomBytes } = naclPkg;
|
|
9
10
|
|
|
@@ -66,9 +67,8 @@ export function sortArrayByObjectKey(arr = [], key) {
|
|
|
66
67
|
};
|
|
67
68
|
|
|
68
69
|
export async function niceHash(str = '') {
|
|
69
|
-
const hash = await sha256(str);
|
|
70
|
-
|
|
71
|
-
return hash;
|
|
70
|
+
const hash = Buffer.from(await sha256(str), 'hex').toString('base64');
|
|
71
|
+
return purifyFilepath(hash);
|
|
72
72
|
};
|
|
73
73
|
|
|
74
74
|
export const sameInstance = (var1, var2) => {
|
package/src/helpers/purger.js
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
|
-
import { openDB, SQLITE_COMMANDS, SQLITE_PATH } from "./sqlite_manager";
|
|
2
1
|
import { Validator } from "guard-object";
|
|
3
2
|
import { incrementDatabaseSizeCore } from "../products/database/counter";
|
|
4
3
|
import { incrementFetcherSizeCore } from "../products/http_callable/counter";
|
|
5
4
|
import unsetLodash from 'lodash/unset';
|
|
6
|
-
import {
|
|
5
|
+
import { FS_PATH, getSystem } from "./fs_manager";
|
|
7
6
|
|
|
8
|
-
const { LIMITER_DATA, LIMITER_RESULT, DB_COUNT_QUERY, FETCH_RESOURCES } =
|
|
7
|
+
const { LIMITER_DATA, LIMITER_RESULT, DB_COUNT_QUERY, FETCH_RESOURCES } = FS_PATH;
|
|
8
|
+
|
|
9
|
+
const inlineFsData = (arr, access_id_node = 'access_id') =>
|
|
10
|
+
arr.map(([access_id, obj]) => {
|
|
11
|
+
obj[access_id_node] = access_id;
|
|
12
|
+
return obj;
|
|
13
|
+
});
|
|
9
14
|
|
|
10
15
|
export const purgeRedundantRecords = async (data, builder) => {
|
|
11
16
|
const { io, maxLocalDatabaseSize = 10485760, maxLocalFetchHttpSize = 10485760 } = builder;
|
|
@@ -145,28 +150,24 @@ export const purgeRedundantRecords = async (data, builder) => {
|
|
|
145
150
|
instances.map(async obj => {
|
|
146
151
|
const { builder, isCounter, path } = obj;
|
|
147
152
|
|
|
148
|
-
const sqlite = await openDB(builder);
|
|
149
|
-
|
|
150
153
|
try {
|
|
151
154
|
if (isCounter) {
|
|
152
|
-
const data =
|
|
153
|
-
|
|
155
|
+
const data = inlineFsData(
|
|
156
|
+
await getSystem(builder).list(DB_COUNT_QUERY(path), ['size', 'touched'])
|
|
154
157
|
);
|
|
155
158
|
return data.map(v => [v, obj]);
|
|
156
159
|
}
|
|
157
160
|
|
|
158
161
|
const [instanceData, resultData] = await Promise.all([
|
|
159
|
-
|
|
160
|
-
|
|
162
|
+
getSystem(builder).list(LIMITER_DATA(path), ['touched', 'size']).catch(() => []),
|
|
163
|
+
getSystem(builder).list(LIMITER_RESULT(path), ['touched', 'size']).catch(() => [])
|
|
161
164
|
]).then(r =>
|
|
162
|
-
r.map(v => v
|
|
165
|
+
r.map((v, i) => inlineFsData(v, i ? 'access_id_limiter' : 'access_id'))
|
|
163
166
|
);
|
|
164
167
|
return [...instanceData, ...resultData].map(v => [v, obj]);
|
|
165
168
|
} catch (error) {
|
|
166
169
|
console.error('redundantDbRanking err:', error);
|
|
167
170
|
return [];
|
|
168
|
-
} finally {
|
|
169
|
-
sqlite.close();
|
|
170
171
|
}
|
|
171
172
|
})
|
|
172
173
|
).then(r =>
|
|
@@ -186,22 +187,15 @@ export const purgeRedundantRecords = async (data, builder) => {
|
|
|
186
187
|
|
|
187
188
|
console.warn(`purging ${cuts} of ${redundantDbRanking.length} db entities`);
|
|
188
189
|
await Promise.all(redundantDbRanking.slice(0, cuts).map(async ([v, { builder, isCounter, path }]) => {
|
|
189
|
-
let db_filename;
|
|
190
|
-
const sqlite = await openDB(builder, n => db_filename = n);
|
|
191
190
|
try {
|
|
192
191
|
const table = (isCounter ? DB_COUNT_QUERY : 'access_id_limiter' in v ? LIMITER_RESULT : LIMITER_DATA)(path);
|
|
193
192
|
const id_field = 'access_id_limiter' in v ? 'access_id_limiter' : 'access_id';
|
|
194
193
|
const primary_key = v[id_field];
|
|
195
|
-
|
|
196
|
-
await Promise.all([
|
|
197
|
-
sqlite.executeSql(SQLITE_COMMANDS.DELETE_ROW(table, `${id_field} = ?`), [primary_key]),
|
|
198
|
-
deleteBigData(getStoreID(db_filename, table, primary_key)).catch(() => null)
|
|
199
|
-
]);
|
|
194
|
+
await getSystem(builder).delete(table, primary_key);
|
|
200
195
|
if (!isCounter) incrementDatabaseSizeCore(data.DatabaseStats, builder, path, -v.size);
|
|
201
196
|
} catch (error) {
|
|
202
197
|
console.log('db redundantClearing err:', error);
|
|
203
198
|
}
|
|
204
|
-
sqlite.close();
|
|
205
199
|
}));
|
|
206
200
|
console.log('database purging complete');
|
|
207
201
|
} catch (error) {
|
|
@@ -212,13 +206,12 @@ export const purgeRedundantRecords = async (data, builder) => {
|
|
|
212
206
|
try {
|
|
213
207
|
if (!Validator.POSITIVE_NUMBER(_fetcher_size) || !maxLocalFetchHttpSize || _fetcher_size < maxLocalFetchHttpSize) return;
|
|
214
208
|
|
|
215
|
-
const redundantFetchRanking = await Promise.all(
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
)
|
|
220
|
-
|
|
221
|
-
})).then(r =>
|
|
209
|
+
const redundantFetchRanking = await Promise.all(
|
|
210
|
+
Object.entries(fetchers).map(async ([projectUrl]) => {
|
|
211
|
+
const data = inlineFsData(await getSystem(FETCH_RESOURCES(projectUrl)).list('main', ['touched', 'size']));
|
|
212
|
+
return data.map(v => [v, projectUrl]);
|
|
213
|
+
})
|
|
214
|
+
).then(r =>
|
|
222
215
|
r.flat().sort(([a], [b]) =>
|
|
223
216
|
a.touched - b.touched
|
|
224
217
|
)
|
|
@@ -236,19 +229,12 @@ export const purgeRedundantRecords = async (data, builder) => {
|
|
|
236
229
|
|
|
237
230
|
console.warn(`purging ${cuts} of ${redundantFetchRanking.length} fetcher entities`);
|
|
238
231
|
await Promise.all(redundantFetchRanking.slice(0, cuts).map(async ([v, projectUrl]) => {
|
|
239
|
-
let db_filename;
|
|
240
|
-
const sqlite = await openDB(FETCH_RESOURCES(projectUrl), n => db_filename = n);
|
|
241
|
-
|
|
242
232
|
try {
|
|
243
|
-
await
|
|
244
|
-
sqlite.executeSql(SQLITE_COMMANDS.DELETE_ROW('main', 'access_id = ?'), [v.access_id]),
|
|
245
|
-
deleteBigData(getStoreID(db_filename, 'main', v.access_id)).catch(() => null)
|
|
246
|
-
]);
|
|
233
|
+
await getSystem(FETCH_RESOURCES(projectUrl)).delete('main', v.access_id);
|
|
247
234
|
incrementFetcherSizeCore(data.DatabaseStats, projectUrl, -v.size);
|
|
248
235
|
} catch (error) {
|
|
249
|
-
console.log('fetcher redundantClearing err:', error);
|
|
236
|
+
console.log('fetcher redundantClearing err:', error, ' obj:', v, ' projectUrl:', projectUrl);
|
|
250
237
|
}
|
|
251
|
-
sqlite.close();
|
|
252
238
|
}));
|
|
253
239
|
console.log('fetcher purging complete');
|
|
254
240
|
} catch (error) {
|
|
@@ -268,8 +254,4 @@ const breakDbMap = (obj, callback) =>
|
|
|
268
254
|
});
|
|
269
255
|
});
|
|
270
256
|
});
|
|
271
|
-
});
|
|
272
|
-
|
|
273
|
-
export const purgeInstance = (projectUrl) => {
|
|
274
|
-
// TODO: purge when signed-out
|
|
275
|
-
}
|
|
257
|
+
});
|
package/src/helpers/utils.js
CHANGED
|
@@ -4,66 +4,54 @@ import { serializeE2E } from "./peripherals";
|
|
|
4
4
|
import { deserializeBSON, serializeToBase64 } from "../products/database/bson";
|
|
5
5
|
import { trySendPendingWrite } from "../products/database";
|
|
6
6
|
import { deserialize } from "entity-serializer";
|
|
7
|
-
import { openDB, SQLITE_COMMANDS, SQLITE_PATH } from "./sqlite_manager";
|
|
8
7
|
import { purgeRedundantRecords } from "./purger";
|
|
9
|
-
import {
|
|
8
|
+
import { FS_PATH, getSystem } from "./fs_manager";
|
|
10
9
|
|
|
11
|
-
const { FILE_NAME, TABLE_NAME } =
|
|
10
|
+
const { FILE_NAME, TABLE_NAME } = FS_PATH;
|
|
12
11
|
|
|
13
12
|
const CacheKeys = Object.keys(CacheStore);
|
|
14
13
|
|
|
15
|
-
export const updateCacheStore = (
|
|
14
|
+
export const updateCacheStore = async (node) => {
|
|
16
15
|
const { io, promoteCache } = Scoped.ReleaseCacheData;
|
|
17
16
|
|
|
18
|
-
const
|
|
19
|
-
|
|
17
|
+
const {
|
|
18
|
+
AuthStore,
|
|
19
|
+
EmulatedAuth,
|
|
20
|
+
PendingAuthPurge,
|
|
21
|
+
DatabaseStore,
|
|
22
|
+
PendingWrites,
|
|
23
|
+
...restStore
|
|
24
|
+
} = CacheStore;
|
|
25
|
+
|
|
26
|
+
if (io) {
|
|
27
|
+
const txt = JSON.stringify({
|
|
20
28
|
AuthStore,
|
|
21
29
|
EmulatedAuth,
|
|
22
30
|
PendingAuthPurge,
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
const sqlite = await openDB(FILE_NAME);
|
|
48
|
-
await Promise.all(
|
|
49
|
-
updationKey
|
|
50
|
-
.map(v => [v, v === 'PendingWrites' ? serializeToBase64(CacheStore[v]) : CacheStore[v]])
|
|
51
|
-
.map(async ([ref, value]) => {
|
|
52
|
-
const blobData = await handleBigData(getStoreID(FILE_NAME, TABLE_NAME, ref), value);
|
|
53
|
-
return sqlite.executeSql(SQLITE_COMMANDS.MERGE(TABLE_NAME, ['ref', 'value']), [ref, blobData]);
|
|
54
|
-
})
|
|
55
|
-
).catch(err => {
|
|
56
|
-
console.error('updateCacheStore err:', err);
|
|
57
|
-
}).finally(() => {
|
|
58
|
-
sqlite.close();
|
|
59
|
-
});
|
|
60
|
-
}
|
|
61
|
-
};
|
|
62
|
-
|
|
63
|
-
clearTimeout(Scoped.cacheStorageReducer);
|
|
64
|
-
if (timer) {
|
|
65
|
-
Scoped.cacheStorageReducer = setTimeout(doUpdate, timer);
|
|
66
|
-
} else doUpdate();
|
|
31
|
+
...promoteCache ? {
|
|
32
|
+
DatabaseStore: serializeToBase64(DatabaseStore),
|
|
33
|
+
PendingWrites: serializeToBase64(PendingWrites)
|
|
34
|
+
} : {},
|
|
35
|
+
...promoteCache ? restStore : {}
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
io.output(txt, node);
|
|
39
|
+
} else {
|
|
40
|
+
// use sqlite
|
|
41
|
+
const exclusion = ['DatabaseStore', 'DatabaseCountResult', 'FetchedStore'];
|
|
42
|
+
const updationKey = (node ? Array.isArray(node) ? node : [node] : CacheKeys).filter(v => !exclusion.includes(v));
|
|
43
|
+
|
|
44
|
+
if (!updationKey.length) return;
|
|
45
|
+
await Promise.all(
|
|
46
|
+
updationKey
|
|
47
|
+
.map(v => [v, v === 'PendingWrites' ? serializeToBase64(CacheStore[v]) : CacheStore[v]])
|
|
48
|
+
.map(([ref, value]) =>
|
|
49
|
+
getSystem(FILE_NAME).set(TABLE_NAME, ref, { value })
|
|
50
|
+
)
|
|
51
|
+
).catch(err => {
|
|
52
|
+
console.error('updateCacheStore err:', err);
|
|
53
|
+
});
|
|
54
|
+
}
|
|
67
55
|
};
|
|
68
56
|
|
|
69
57
|
export const releaseCacheStore = async (builder) => {
|
|
@@ -75,19 +63,15 @@ export const releaseCacheStore = async (builder) => {
|
|
|
75
63
|
if (io) {
|
|
76
64
|
data = JSON.parse((await io.input()) || '{}');
|
|
77
65
|
} else {
|
|
78
|
-
const sqlite = await openDB(FILE_NAME);
|
|
79
|
-
await sqlite.executeSql(`CREATE TABLE IF NOT EXISTS ${TABLE_NAME} ( ref TEXT PRIMARY KEY, value BLOB )`).catch(() => null);
|
|
80
66
|
try {
|
|
81
|
-
const
|
|
67
|
+
const query = await getSystem(FILE_NAME).list(TABLE_NAME, ['value']).catch(() => []);
|
|
82
68
|
data = Object.fromEntries(
|
|
83
|
-
|
|
84
|
-
[
|
|
85
|
-
)
|
|
69
|
+
query.map(([ref, { value }]) =>
|
|
70
|
+
[ref, value]
|
|
71
|
+
)
|
|
86
72
|
);
|
|
87
73
|
} catch (error) {
|
|
88
74
|
console.error('initializeCache sqlite data release err:', error);
|
|
89
|
-
} finally {
|
|
90
|
-
sqlite.close();
|
|
91
75
|
}
|
|
92
76
|
}
|
|
93
77
|
await purgeRedundantRecords(data, builder);
|
package/src/helpers/variables.js
CHANGED
|
@@ -8,7 +8,6 @@ export const Scoped = {
|
|
|
8
8
|
InitializedProject: {},
|
|
9
9
|
ReleaseCacheData: undefined,
|
|
10
10
|
AuthJWTToken: {},
|
|
11
|
-
cacheStorageReducer: undefined,
|
|
12
11
|
IsStoreReady: false,
|
|
13
12
|
TokenRefreshTimer: {},
|
|
14
13
|
LastTokenRefreshRef: {},
|
|
@@ -26,11 +25,6 @@ export const Scoped = {
|
|
|
26
25
|
database: {},
|
|
27
26
|
dbQueryCount: {},
|
|
28
27
|
httpFetch: {}
|
|
29
|
-
},
|
|
30
|
-
initedSqliteInstances: {
|
|
31
|
-
httpFetch: {},
|
|
32
|
-
dbQueryCount: {},
|
|
33
|
-
database: {}
|
|
34
28
|
}
|
|
35
29
|
};
|
|
36
30
|
|
|
@@ -62,6 +56,9 @@ export const CacheStore = {
|
|
|
62
56
|
},
|
|
63
57
|
AuthStore: {},
|
|
64
58
|
PendingAuthPurge: {},
|
|
59
|
+
/**
|
|
60
|
+
* [the instance url]: the url been emulated
|
|
61
|
+
*/
|
|
65
62
|
EmulatedAuth: {},
|
|
66
63
|
PendingWrites: {},
|
|
67
64
|
FetchedStore: {}
|
package/src/index.d.ts
CHANGED
|
@@ -96,11 +96,7 @@ interface ReleaseCacheOption_IO {
|
|
|
96
96
|
|
|
97
97
|
interface ReleaseCacheOption {
|
|
98
98
|
/**
|
|
99
|
-
*
|
|
100
|
-
*/
|
|
101
|
-
sqliteKey?: string;
|
|
102
|
-
/**
|
|
103
|
-
* sqlite is used as the default caching mechanism
|
|
99
|
+
* fs is used as the default caching mechanism
|
|
104
100
|
*
|
|
105
101
|
* providing `io` means you want to override this behaviour and handle caching yourself in the system's memory
|
|
106
102
|
*/
|
package/src/index.js
CHANGED
|
@@ -419,10 +419,7 @@ const discloseSocketArguments = (args = []) => {
|
|
|
419
419
|
|
|
420
420
|
const validateReleaseCacheProp = (prop) => {
|
|
421
421
|
Object.entries(prop).forEach(([k, v]) => {
|
|
422
|
-
if (k === '
|
|
423
|
-
if (typeof v !== 'string' || v.trim().length <= 0)
|
|
424
|
-
throw `Invalid value supplied to "sqliteKey", value must be a string and greater than 0 characters`;
|
|
425
|
-
} else if (k === 'io') {
|
|
422
|
+
if (k === 'io') {
|
|
426
423
|
Object.entries(v).forEach(([k, v]) => {
|
|
427
424
|
if (k === 'input' || k === 'output') {
|
|
428
425
|
if (typeof v !== 'function')
|
|
@@ -23,7 +23,7 @@ export const injectFreshToken = async (config, { token, refreshToken }) => {
|
|
|
23
23
|
const isEmulated = projectUrl in CacheStore.EmulatedAuth;
|
|
24
24
|
if (isEmulated) delete CacheStore.EmulatedAuth[projectUrl];
|
|
25
25
|
|
|
26
|
-
updateCacheStore(
|
|
26
|
+
updateCacheStore(['AuthStore', isEmulated ? 'EmulatedAuth' : ''].filter(v => v));
|
|
27
27
|
|
|
28
28
|
triggerAuthToken(projectUrl);
|
|
29
29
|
initTokenRefresher(config);
|
|
@@ -47,7 +47,7 @@ export const injectEmulatedAuth = async (config, emulatedURL) => {
|
|
|
47
47
|
Scoped.AuthJWTToken[projectUrl] = token;
|
|
48
48
|
CacheStore.EmulatedAuth[projectUrl] = emulatedURL;
|
|
49
49
|
|
|
50
|
-
updateCacheStore(
|
|
50
|
+
updateCacheStore(['AuthStore', 'EmulatedAuth']);
|
|
51
51
|
triggerAuthToken(projectUrl);
|
|
52
52
|
initTokenRefresher(config);
|
|
53
53
|
};
|
|
@@ -161,7 +161,7 @@ const refreshToken = (builder, processRef, remainRetries = 1, isForceRefresh) =>
|
|
|
161
161
|
triggerAuthToken(v, isInit);
|
|
162
162
|
if (isForceRefresh) Scoped.InitiatedForcedToken[v] = true;
|
|
163
163
|
});
|
|
164
|
-
updateCacheStore(
|
|
164
|
+
updateCacheStore(['AuthStore']);
|
|
165
165
|
initTokenRefresher(builder);
|
|
166
166
|
} else reject(lostProcess.simpleError);
|
|
167
167
|
} catch (e) {
|
|
@@ -232,13 +232,12 @@ const doCustomSignup = (builder, email, password, name, metadata) => new Promise
|
|
|
232
232
|
|
|
233
233
|
const purgeCache = (url, isMain) => {
|
|
234
234
|
if (url in Scoped.AuthJWTToken) delete Scoped.AuthJWTToken[url];
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
}
|
|
235
|
+
[
|
|
236
|
+
isMain ? 'EmulatedAuth' : undefined,
|
|
237
|
+
'AuthStore',
|
|
238
|
+
'PendingWrites'
|
|
239
|
+
].forEach(e => {
|
|
240
|
+
if (e && CacheStore[e][url]) delete CacheStore[e][url];
|
|
242
241
|
});
|
|
243
242
|
TokenRefreshListener.dispatch(url);
|
|
244
243
|
triggerAuthToken(url);
|
|
@@ -257,7 +256,7 @@ export const doSignOut = async (builder) => {
|
|
|
257
256
|
const emulatedURL = CacheStore.EmulatedAuth[builder.projectUrl];
|
|
258
257
|
|
|
259
258
|
clearCacheForSignout(builder, !emulatedURL);
|
|
260
|
-
updateCacheStore(
|
|
259
|
+
updateCacheStore(['AuthStore', 'EmulatedAuth']);
|
|
261
260
|
if (emulatedURL) return;
|
|
262
261
|
await revokeAuthIntance(builder);
|
|
263
262
|
};
|
|
@@ -304,7 +303,7 @@ export const purgePendingToken = async (nodeId) => {
|
|
|
304
303
|
throw simplifyCaughtError(e).simpleError;
|
|
305
304
|
} finally {
|
|
306
305
|
delete CacheStore.PendingAuthPurge[nodeId];
|
|
307
|
-
updateCacheStore(
|
|
306
|
+
updateCacheStore(['PendingAuthPurge']);
|
|
308
307
|
}
|
|
309
308
|
};
|
|
310
309
|
|
|
@@ -12,10 +12,9 @@ import { niceGuard, Validator } from "guard-object";
|
|
|
12
12
|
import { TIMESTAMP } from "../..";
|
|
13
13
|
import { docSize, incrementDatabaseSize } from "./counter";
|
|
14
14
|
import { deserializeBSON, serializeToBase64 } from "./bson";
|
|
15
|
-
import {
|
|
16
|
-
import { getStoreID, handleBigData, parseBigData } from "../../helpers/fs_manager";
|
|
15
|
+
import { FS_PATH, getSystem, useFS } from "../../helpers/fs_manager";
|
|
17
16
|
|
|
18
|
-
const { LIMITER_DATA, LIMITER_RESULT, DB_COUNT_QUERY } =
|
|
17
|
+
const { LIMITER_DATA, LIMITER_RESULT, DB_COUNT_QUERY } = FS_PATH;
|
|
19
18
|
|
|
20
19
|
export const listenQueryEntry = (callback, { accessId, builder, config, processId }) => {
|
|
21
20
|
const { projectUrl, dbName, dbUrl, path } = builder;
|
|
@@ -50,28 +49,14 @@ export const insertCountQuery = async (builder, access_id, value) => {
|
|
|
50
49
|
const { io } = Scoped.ReleaseCacheData;
|
|
51
50
|
if (io) {
|
|
52
51
|
setLodash(CacheStore.DatabaseCountResult, [projectUrl, dbUrl, dbName, path, access_id], { value, touched: Date.now() });
|
|
52
|
+
updateCacheStore(['DatabaseCountResult']);
|
|
53
53
|
} else {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
if (!Scoped.initedSqliteInstances.dbQueryCount[initNode]) {
|
|
57
|
-
Scoped.initedSqliteInstances.dbQueryCount[initNode] = (async () => {
|
|
58
|
-
await sqlite.executeSql(`CREATE TABLE IF NOT EXISTS ${DB_COUNT_QUERY(path)} ( access_id TEXT PRIMARY KEY, value TEXT, touched INTEGER )`).catch(() => null);
|
|
59
|
-
await Promise.allSettled([
|
|
60
|
-
// sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX(DB_COUNT_QUERY(path), ['access_id'])),
|
|
61
|
-
// sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX(DB_COUNT_QUERY(path), ['touched']))
|
|
62
|
-
]);
|
|
63
|
-
})();
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
await Scoped.initedSqliteInstances.dbQueryCount[initNode];
|
|
67
|
-
await sqlite.executeSql(
|
|
68
|
-
SQLITE_COMMANDS.MERGE(DB_COUNT_QUERY(path), ['access_id', 'value', 'touched']),
|
|
69
|
-
[access_id, JSON.stringify(value), Date.now()]
|
|
70
|
-
);
|
|
54
|
+
await useFS(builder, access_id, 'dbQueryCount')(async fs => {
|
|
55
|
+
await fs.set(DB_COUNT_QUERY(path), access_id, { value, touched: Date.now() });
|
|
71
56
|
setLodash(CacheStore.DatabaseStats.counters, [projectUrl, dbUrl, dbName, path], true);
|
|
72
57
|
});
|
|
58
|
+
updateCacheStore(['DatabaseStats']);
|
|
73
59
|
}
|
|
74
|
-
updateCacheStore(undefined, ['DatabaseCountResult'])
|
|
75
60
|
}
|
|
76
61
|
|
|
77
62
|
export const getCountQuery = async (builder, access_id) => {
|
|
@@ -83,14 +68,13 @@ export const getCountQuery = async (builder, access_id) => {
|
|
|
83
68
|
if (data) data.touched = Date.now();
|
|
84
69
|
return data && data.value;
|
|
85
70
|
} else {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
await
|
|
90
|
-
return
|
|
91
|
-
}
|
|
92
|
-
);
|
|
93
|
-
return result;
|
|
71
|
+
return useFS(builder, access_id, 'dbQueryCount')(async fs => {
|
|
72
|
+
const data = await fs.find(DB_COUNT_QUERY(path), access_id, ['value']).catch(() => null);
|
|
73
|
+
if (data) {
|
|
74
|
+
await fs.set(DB_COUNT_QUERY(path), access_id, { touched: Date.now() });
|
|
75
|
+
return data.value;
|
|
76
|
+
}
|
|
77
|
+
});
|
|
94
78
|
}
|
|
95
79
|
}
|
|
96
80
|
|
|
@@ -106,35 +90,13 @@ export const insertRecord = async (builder, config, accessIdWithoutLimit, value,
|
|
|
106
90
|
const thisSize = docSize(value);
|
|
107
91
|
|
|
108
92
|
if (!io) {
|
|
109
|
-
await
|
|
110
|
-
const initNode = `${projectUrl}_${dbUrl}_${dbName}_${path}`;
|
|
111
|
-
|
|
112
|
-
if (!Scoped.initedSqliteInstances.database[initNode]) {
|
|
113
|
-
Scoped.initedSqliteInstances.database[initNode] = (async () => {
|
|
114
|
-
await Promise.allSettled([
|
|
115
|
-
sqlite.executeSql(`CREATE TABLE IF NOT EXISTS ${LIMITER_DATA(path)} ( access_id TEXT PRIMARY KEY, value BLOB, touched INTEGER, size INTEGER )`),
|
|
116
|
-
sqlite.executeSql(`CREATE TABLE IF NOT EXISTS ${LIMITER_RESULT(path)} ( access_id_limiter TEXT PRIMARY KEY, access_id TEXT, value BLOB, touched INTEGER, size INTEGER )`)
|
|
117
|
-
]);
|
|
118
|
-
|
|
119
|
-
await Promise.allSettled([
|
|
120
|
-
// sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX(LIMITER_DATA(path), ['access_id'])),
|
|
121
|
-
// sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX(LIMITER_DATA(path), ['touched'])),
|
|
122
|
-
// sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX(LIMITER_RESULT(path), ['access_id_limiter'])),
|
|
123
|
-
// sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX(LIMITER_RESULT(path), ['touched']))
|
|
124
|
-
]);
|
|
125
|
-
})();
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
await Scoped.initedSqliteInstances.database[initNode];
|
|
129
|
-
|
|
93
|
+
await useFS(builder, accessIdWithoutLimit, 'database')(async fs => {
|
|
130
94
|
const resultAccessId = `${accessIdWithoutLimit}-${limit}`;
|
|
131
95
|
|
|
132
96
|
const [instanceData, resultData] = await Promise.all([
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
])
|
|
136
|
-
r.map(v => v[0].rows.item(0))
|
|
137
|
-
);
|
|
97
|
+
fs.find(LIMITER_DATA(path), accessIdWithoutLimit, ['size']).catch(() => undefined),
|
|
98
|
+
fs.find(LIMITER_RESULT(path), resultAccessId, ['size']).catch(() => undefined)
|
|
99
|
+
]);
|
|
138
100
|
const isEpisode = episode === 1 || !!resultData;
|
|
139
101
|
|
|
140
102
|
const editionSizeOffset = thisSize - (instanceData?.size || 0);
|
|
@@ -151,27 +113,24 @@ export const insertRecord = async (builder, config, accessIdWithoutLimit, value,
|
|
|
151
113
|
data: value,
|
|
152
114
|
size: thisSize
|
|
153
115
|
});
|
|
154
|
-
const [blobData, episodeBlobData] = await Promise.all([
|
|
155
|
-
handleBigData(getStoreID(db_filename, LIMITER_DATA(path), accessIdWithoutLimit), newData),
|
|
156
|
-
isEpisode ?
|
|
157
|
-
handleBigData(getStoreID(db_filename, LIMITER_RESULT(path), resultAccessId), newResultData)
|
|
158
|
-
: Promise.resolve()
|
|
159
|
-
]);
|
|
160
116
|
|
|
161
117
|
await Promise.all([
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
118
|
+
fs.set(LIMITER_DATA(path), accessIdWithoutLimit, {
|
|
119
|
+
value: newData,
|
|
120
|
+
touched: Date.now(),
|
|
121
|
+
size: thisSize
|
|
122
|
+
}),
|
|
166
123
|
isEpisode ?
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
124
|
+
fs.set(LIMITER_RESULT(path), resultAccessId, {
|
|
125
|
+
access_id: accessIdWithoutLimit,
|
|
126
|
+
value: newResultData,
|
|
127
|
+
touched: Date.now(),
|
|
128
|
+
size: thisSize
|
|
129
|
+
}) : Promise.resolve()
|
|
171
130
|
]);
|
|
172
131
|
incrementDatabaseSize(builder, path, editionSizeOffset + resultSizeOffset);
|
|
173
132
|
});
|
|
174
|
-
updateCacheStore(
|
|
133
|
+
updateCacheStore(['DatabaseStats']);
|
|
175
134
|
return;
|
|
176
135
|
}
|
|
177
136
|
|
|
@@ -200,7 +159,7 @@ export const insertRecord = async (builder, config, accessIdWithoutLimit, value,
|
|
|
200
159
|
|
|
201
160
|
setLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'instance', accessIdWithoutLimit], newData);
|
|
202
161
|
if (isEpisode) setLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'episode', accessIdWithoutLimit, limit], cloneDeep(newResultData));
|
|
203
|
-
updateCacheStore(
|
|
162
|
+
updateCacheStore(['DatabaseStore', 'DatabaseStats']);
|
|
204
163
|
};
|
|
205
164
|
|
|
206
165
|
export const getRecord = async (builder, accessIdWithoutLimit, episode = 0) => {
|
|
@@ -231,19 +190,19 @@ export const getRecord = async (builder, accessIdWithoutLimit, episode = 0) => {
|
|
|
231
190
|
}
|
|
232
191
|
|
|
233
192
|
if (!io) {
|
|
234
|
-
const record = await
|
|
193
|
+
const record = await useFS(builder, accessIdWithoutLimit, 'database')(async fs => {
|
|
235
194
|
const resultAccessId = `${accessIdWithoutLimit}-${limit}`;
|
|
236
195
|
|
|
237
196
|
const qData = await (
|
|
238
|
-
isEpisode ?
|
|
239
|
-
|
|
240
|
-
).
|
|
241
|
-
const thisData = qData && deserializeBSON(
|
|
197
|
+
isEpisode ? fs.find(LIMITER_RESULT(path), resultAccessId, ['value']) :
|
|
198
|
+
fs.find(LIMITER_DATA(path), accessIdWithoutLimit, ['value'])
|
|
199
|
+
).catch(() => null);
|
|
200
|
+
const thisData = qData && deserializeBSON(qData.value, true);
|
|
242
201
|
|
|
243
202
|
if (!thisData) return null;
|
|
244
203
|
|
|
245
204
|
if (isEpisode) {
|
|
246
|
-
await
|
|
205
|
+
await fs.set(LIMITER_RESULT(path), resultAccessId, { touched: Date.now() });
|
|
247
206
|
return [thisData.data];
|
|
248
207
|
}
|
|
249
208
|
|
|
@@ -253,7 +212,7 @@ export const getRecord = async (builder, accessIdWithoutLimit, episode = 0) => {
|
|
|
253
212
|
latest_limiter === undefined ||
|
|
254
213
|
(Validator.POSITIVE_NUMBER(limit) && latest_limiter >= limit)
|
|
255
214
|
) {
|
|
256
|
-
await
|
|
215
|
+
await fs.set(LIMITER_DATA(path), accessIdWithoutLimit, { touched: Date.now() });
|
|
257
216
|
return [transformData(data)];
|
|
258
217
|
}
|
|
259
218
|
});
|
|
@@ -457,46 +416,28 @@ export const addPendingWrites = async (builder, writeId, result) => {
|
|
|
457
416
|
)
|
|
458
417
|
);
|
|
459
418
|
} else {
|
|
460
|
-
const
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
)
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
)
|
|
483
|
-
);
|
|
484
|
-
const editedBlobData = await handleBigData(
|
|
485
|
-
getStoreID(db_filename, LIMITER_DATA(path), access_id),
|
|
486
|
-
serializeToBase64(data[1])
|
|
487
|
-
);
|
|
488
|
-
|
|
489
|
-
await sqlite.executeSql(
|
|
490
|
-
SQLITE_COMMANDS.MERGE(LIMITER_DATA(path), ['access_id', 'value', 'touched', 'size']),
|
|
491
|
-
[access_id, editedBlobData, Date.now(), data[1].size]
|
|
492
|
-
);
|
|
493
|
-
})
|
|
494
|
-
));
|
|
495
|
-
} catch (error) {
|
|
496
|
-
throw error;
|
|
497
|
-
} finally {
|
|
498
|
-
sqlite.close();
|
|
499
|
-
}
|
|
419
|
+
const colListing = await getSystem(builder).list(LIMITER_DATA(path), []).catch(() => []);
|
|
420
|
+
const pathFinder = {};
|
|
421
|
+
|
|
422
|
+
await Promise.all(colListing.map(async ([access_id]) =>
|
|
423
|
+
useFS(builder, access_id, 'database')(async fs => {
|
|
424
|
+
const data = await fs.find(LIMITER_DATA(path), access_id, ['value'])
|
|
425
|
+
.then(r => deserializeBSON(r.value, true));
|
|
426
|
+
|
|
427
|
+
await MutateDataInstance([access_id, data], path =>
|
|
428
|
+
pathFinder[path] || (
|
|
429
|
+
pathFinder[path] = fs.list(LIMITER_DATA(path), ['value'])
|
|
430
|
+
.then(v => v.map(d => deserializeBSON(d[1].value, true).data).flat())
|
|
431
|
+
.catch(() => [])
|
|
432
|
+
)
|
|
433
|
+
);
|
|
434
|
+
await fs.set(LIMITER_DATA(path), access_id, {
|
|
435
|
+
touched: Date.now(),
|
|
436
|
+
value: serializeToBase64(data),
|
|
437
|
+
size: data.size
|
|
438
|
+
});
|
|
439
|
+
})
|
|
440
|
+
));
|
|
500
441
|
}
|
|
501
442
|
|
|
502
443
|
async function MutateDataInstance([entityId, dataObj], pathGetter) {
|
|
@@ -706,7 +647,7 @@ export const addPendingWrites = async (builder, writeId, result) => {
|
|
|
706
647
|
addedOn: Date.now()
|
|
707
648
|
}));
|
|
708
649
|
|
|
709
|
-
updateCacheStore(
|
|
650
|
+
updateCacheStore(['DatabaseStore', 'PendingWrites', 'DatabaseStats']);
|
|
710
651
|
notifyDatabaseNodeChanges(builder, [...pathChanges]);
|
|
711
652
|
};
|
|
712
653
|
|
|
@@ -724,22 +665,17 @@ export const removePendingWrite = async (builder, writeId, revert) => {
|
|
|
724
665
|
if (io) {
|
|
725
666
|
RevertMutation(getLodash(CacheStore.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'instance', access_id]));
|
|
726
667
|
} else {
|
|
727
|
-
await
|
|
728
|
-
const colObj = await
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
}).catch(() => null);
|
|
668
|
+
await useFS(builder, access_id, 'database')(async fs => {
|
|
669
|
+
const colObj = await fs.find(LIMITER_DATA(path), access_id, ['value'])
|
|
670
|
+
.then(v => deserializeBSON(v.value))
|
|
671
|
+
.catch(() => null);
|
|
732
672
|
if (!colObj) return;
|
|
733
673
|
RevertMutation(colObj);
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
await sqlite.executeSql(
|
|
740
|
-
SQLITE_COMMANDS.MERGE(LIMITER_DATA(path), ['access_id', 'value', 'touched', 'size']),
|
|
741
|
-
[access_id, revertedBlobData, Date.now(), colObj.size]
|
|
742
|
-
);
|
|
674
|
+
await fs.set(LIMITER_DATA(path), access_id, {
|
|
675
|
+
value: serializeToBase64(colObj),
|
|
676
|
+
touched: Date.now(),
|
|
677
|
+
size: colObj.size
|
|
678
|
+
});
|
|
743
679
|
});
|
|
744
680
|
}
|
|
745
681
|
|
|
@@ -782,7 +718,7 @@ export const removePendingWrite = async (builder, writeId, revert) => {
|
|
|
782
718
|
}
|
|
783
719
|
|
|
784
720
|
unsetLodash(CacheStore.PendingWrites, [projectUrl, writeId]);
|
|
785
|
-
updateCacheStore(
|
|
721
|
+
updateCacheStore(['PendingWrites', 'DatabaseStore', 'DatabaseStats']);
|
|
786
722
|
notifyDatabaseNodeChanges(builder, [...pathChanges]);
|
|
787
723
|
};
|
|
788
724
|
|
|
@@ -396,7 +396,7 @@ const countCollection = async (builder, config) => {
|
|
|
396
396
|
|
|
397
397
|
if (e?.simpleError) {
|
|
398
398
|
finalize(undefined, e.simpleError);
|
|
399
|
-
} else if (!disableCache &&
|
|
399
|
+
} else if (!disableCache && Validator.NUMBER(b4Data)) {
|
|
400
400
|
finalize(b4Data);
|
|
401
401
|
} else if (retries > maxRetries) {
|
|
402
402
|
finalize(undefined, { error: 'retry_limit_exceeded', message: `retry exceed limit(${maxRetries})` });
|
|
@@ -2,11 +2,10 @@ import { updateCacheStore } from "../../helpers/utils";
|
|
|
2
2
|
import { CacheStore, Scoped } from "../../helpers/variables";
|
|
3
3
|
import cloneDeep from "lodash/cloneDeep";
|
|
4
4
|
import { serialize } from "entity-serializer";
|
|
5
|
-
import { SQLITE_COMMANDS, SQLITE_PATH, useSqliteLinearAccessId } from "../../helpers/sqlite_manager";
|
|
6
5
|
import { incrementFetcherSize } from "./counter";
|
|
7
|
-
import {
|
|
6
|
+
import { FS_PATH, useFS } from "../../helpers/fs_manager";
|
|
8
7
|
|
|
9
|
-
const { FETCH_RESOURCES } =
|
|
8
|
+
const { FETCH_RESOURCES } = FS_PATH;
|
|
10
9
|
|
|
11
10
|
export const insertFetchResources = async (projectUrl, access_id, value) => {
|
|
12
11
|
value = cloneDeep(value);
|
|
@@ -23,35 +22,16 @@ export const insertFetchResources = async (projectUrl, access_id, value) => {
|
|
|
23
22
|
data: value,
|
|
24
23
|
size: dataSize
|
|
25
24
|
};
|
|
25
|
+
updateCacheStore(['FetchedStore', 'DatabaseStats']);
|
|
26
26
|
} else {
|
|
27
|
-
|
|
27
|
+
await useFS(FETCH_RESOURCES(projectUrl), access_id, 'httpFetch')(async fs => {
|
|
28
|
+
const b4Data = await fs.find('main', access_id, ['size']).catch(() => null);
|
|
28
29
|
|
|
29
|
-
|
|
30
|
-
if (!Scoped.initedSqliteInstances.httpFetch[initNode]) {
|
|
31
|
-
Scoped.initedSqliteInstances.httpFetch[initNode] = (async () => {
|
|
32
|
-
await sqlite.executeSql(`CREATE TABLE IF NOT EXISTS main ( access_id TEXT PRIMARY KEY, value BLOB, touched INTEGER, size INTEGER )`).catch(() => null);
|
|
33
|
-
await Promise.allSettled([
|
|
34
|
-
// sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX('main', ['access_id'])),
|
|
35
|
-
// sqlite.executeSql(SQLITE_COMMANDS.CREATE_INDEX('main', ['touched']))
|
|
36
|
-
]);
|
|
37
|
-
})();
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
await Scoped.initedSqliteInstances.httpFetch[initNode];
|
|
41
|
-
const b4Data = await sqlite.executeSql(`SELECT access_id, size FROM main WHERE access_id = ?`, [access_id]).then(r =>
|
|
42
|
-
r[0].rows.item(0)
|
|
43
|
-
);
|
|
44
|
-
const blobData = await handleBigData(getStoreID(db_filename, 'main', access_id), value);
|
|
45
|
-
|
|
46
|
-
await sqlite.executeSql(
|
|
47
|
-
SQLITE_COMMANDS.MERGE('main', ['access_id', 'value', 'touched', 'size']),
|
|
48
|
-
[access_id, blobData, Date.now(), dataSize]
|
|
49
|
-
);
|
|
30
|
+
await fs.set('main', access_id, { value, size: dataSize, touched: Date.now() });
|
|
50
31
|
incrementFetcherSize(projectUrl, dataSize - (b4Data?.size || 0));
|
|
51
32
|
});
|
|
33
|
+
updateCacheStore(['DatabaseStats']);
|
|
52
34
|
}
|
|
53
|
-
|
|
54
|
-
updateCacheStore(undefined, ['FetchedStore']);
|
|
55
35
|
}
|
|
56
36
|
|
|
57
37
|
export const getFetchResources = async (projectUrl, access_id) => {
|
|
@@ -63,15 +43,12 @@ export const getFetchResources = async (projectUrl, access_id) => {
|
|
|
63
43
|
return record && cloneDeep(record?.data);
|
|
64
44
|
}
|
|
65
45
|
|
|
66
|
-
const res = await
|
|
67
|
-
const query = await
|
|
68
|
-
|
|
69
|
-
const rawData = query && query[0].rows.item(0)?.value;
|
|
70
|
-
if (!rawData) return null;
|
|
46
|
+
const res = await useFS(FETCH_RESOURCES(projectUrl), access_id, 'httpFetch')(async fs => {
|
|
47
|
+
const query = await fs.find('main', access_id, ['value']).catch(() => null);
|
|
48
|
+
if (!query) return null;
|
|
71
49
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
return data;
|
|
50
|
+
await fs.set('main', access_id, { touched: Date.now() });
|
|
51
|
+
return query.value;
|
|
75
52
|
});
|
|
76
53
|
return res;
|
|
77
54
|
}
|
|
@@ -106,7 +106,7 @@ export const mfetch = async (input = '', init, config) => {
|
|
|
106
106
|
|
|
107
107
|
await awaitStore();
|
|
108
108
|
const resolveCache = (reqData) => {
|
|
109
|
-
finalize(buildFetchData(reqData
|
|
109
|
+
finalize(buildFetchData(reqData, { fromCache: true }));
|
|
110
110
|
};
|
|
111
111
|
|
|
112
112
|
try {
|
|
@@ -1,144 +0,0 @@
|
|
|
1
|
-
import { enablePromise, openDatabase } from 'react-native-sqlite-storage';
|
|
2
|
-
import { Scoped, SqliteCollective } from './variables';
|
|
3
|
-
import { niceHash } from './peripherals';
|
|
4
|
-
|
|
5
|
-
enablePromise(true);
|
|
6
|
-
let sqliteKeyHash;
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* this method implement a centralize approach for opening and closing of sqlite database to ensure consistency across multiple task opening and closing the database in diferent order
|
|
10
|
-
*
|
|
11
|
-
* @param {string} name
|
|
12
|
-
* @returns {Promise<import('react-native-sqlite-storage').SQLiteDatabase>}
|
|
13
|
-
*/
|
|
14
|
-
export const openDB = async (name, onName) => {
|
|
15
|
-
|
|
16
|
-
if (name?.projectUrl) {
|
|
17
|
-
const { projectUrl, dbUrl, dbName } = name;
|
|
18
|
-
name = encodeURIComponent(`${projectUrl}_${dbUrl}_${dbName}`) + '.db';
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
const { sqliteKey } = Scoped.ReleaseCacheData;
|
|
22
|
-
|
|
23
|
-
if (sqliteKey) {
|
|
24
|
-
const thisHash = await (sqliteKeyHash || (sqliteKeyHash = niceHash(sqliteKey)));
|
|
25
|
-
name = `${thisHash}__${name}`;
|
|
26
|
-
}
|
|
27
|
-
onName?.(name);
|
|
28
|
-
|
|
29
|
-
if (!SqliteCollective.openedDb[name]) {
|
|
30
|
-
SqliteCollective.openedDbProcess[name] = 0;
|
|
31
|
-
SqliteCollective.openedDb[name] = Promise.allSettled([SqliteCollective.closeDbPromises[name] || Promise.resolve()]).then(() =>
|
|
32
|
-
openDatabase({
|
|
33
|
-
location: 'default',
|
|
34
|
-
name,
|
|
35
|
-
key: sqliteKey
|
|
36
|
-
}).then(db => {
|
|
37
|
-
const prevClose = db.close.bind(db);
|
|
38
|
-
|
|
39
|
-
db.close = () => new Promise((resolve, reject) => {
|
|
40
|
-
if (--SqliteCollective.openedDbProcess[name] === 0) {
|
|
41
|
-
let willClose;
|
|
42
|
-
const timer = setTimeout(async () => {
|
|
43
|
-
willClose = true;
|
|
44
|
-
delete SqliteCollective.openedDb[name];
|
|
45
|
-
delete SqliteCollective.openedDbProcess[name];
|
|
46
|
-
SqliteCollective.closeDbPromises[name] = prevClose().then(() => {
|
|
47
|
-
resolve('active');
|
|
48
|
-
}).catch(e => {
|
|
49
|
-
reject(new Error(`${e}`));
|
|
50
|
-
}).finally(() => {
|
|
51
|
-
delete SqliteCollective.closeDbPromises[name];
|
|
52
|
-
});
|
|
53
|
-
delete SqliteCollective.openedDbReducerTimer[name];
|
|
54
|
-
}, 7);
|
|
55
|
-
|
|
56
|
-
SqliteCollective.openedDbReducerTimer[name] = () => {
|
|
57
|
-
clearTimeout(timer);
|
|
58
|
-
if (!willClose) resolve('passive');
|
|
59
|
-
delete SqliteCollective.openedDbReducerTimer[name];
|
|
60
|
-
}
|
|
61
|
-
} else resolve('passive');
|
|
62
|
-
});
|
|
63
|
-
return db;
|
|
64
|
-
})
|
|
65
|
-
);
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
SqliteCollective.openedDbReducerTimer[name]?.();
|
|
69
|
-
++SqliteCollective.openedDbProcess[name];
|
|
70
|
-
const thisDb = await SqliteCollective.openedDb[name];
|
|
71
|
-
let hasClosed;
|
|
72
|
-
|
|
73
|
-
const thisClose = async () => {
|
|
74
|
-
if (hasClosed) return;
|
|
75
|
-
hasClosed = true;
|
|
76
|
-
return (await thisDb.close());
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
return new Proxy({}, {
|
|
80
|
-
get: (_, n) => {
|
|
81
|
-
if (n === 'close') {
|
|
82
|
-
return thisClose;
|
|
83
|
-
} else if (typeof thisDb[n] === 'function')
|
|
84
|
-
return thisDb[n].bind(thisDb);
|
|
85
|
-
return thisDb[n];
|
|
86
|
-
},
|
|
87
|
-
set: (_, n, v) => {
|
|
88
|
-
thisDb[n] = v;
|
|
89
|
-
}
|
|
90
|
-
});
|
|
91
|
-
};
|
|
92
|
-
|
|
93
|
-
/**
|
|
94
|
-
* this method linearize read/write on sqlite ensuring consistency across concurrent operations
|
|
95
|
-
*
|
|
96
|
-
* @param {any} builder
|
|
97
|
-
* @param {string} access_id
|
|
98
|
-
* @param {'database' | 'dbQueryCount' | 'httpFetch'} node
|
|
99
|
-
* @returns {(task: (sqlite: import("react-native-sqlite-storage").SQLiteDatabase, db_filename: string) => Promise<{any}> )=> Promise<{any}>}
|
|
100
|
-
*/
|
|
101
|
-
export const useSqliteLinearAccessId = (builder, access_id, node) => async (task) => {
|
|
102
|
-
const { projectUrl, dbUrl, dbName } = builder;
|
|
103
|
-
const nodeId = typeof builder === 'string' ? `${builder}_${access_id}` : `${projectUrl}_${dbUrl}_${dbName}_${access_id}`;
|
|
104
|
-
let db_filename;
|
|
105
|
-
|
|
106
|
-
const sqlite = await openDB(builder, n => db_filename = n);
|
|
107
|
-
|
|
108
|
-
const thatProcess = Scoped.linearSqliteProcess[node][nodeId];
|
|
109
|
-
|
|
110
|
-
const thisPromise = new Promise(async (resolve, reject) => {
|
|
111
|
-
try {
|
|
112
|
-
if (thatProcess !== undefined) await thatProcess;
|
|
113
|
-
} catch (_) { }
|
|
114
|
-
try {
|
|
115
|
-
resolve(await task(sqlite, db_filename));
|
|
116
|
-
} catch (error) {
|
|
117
|
-
console.error('useSqliteLinearAccessId err:', error, ' builder:', builder);
|
|
118
|
-
reject(error);
|
|
119
|
-
} finally {
|
|
120
|
-
if (Scoped.linearSqliteProcess[node][nodeId] === thisPromise)
|
|
121
|
-
delete Scoped.linearSqliteProcess[node][nodeId];
|
|
122
|
-
sqlite.close();
|
|
123
|
-
}
|
|
124
|
-
});
|
|
125
|
-
|
|
126
|
-
Scoped.linearSqliteProcess[node][nodeId] = thisPromise;
|
|
127
|
-
return (await thisPromise);
|
|
128
|
-
};
|
|
129
|
-
|
|
130
|
-
export const SQLITE_PATH = {
|
|
131
|
-
FILE_NAME: 'MOSQUITO_TRANSPORT.db',
|
|
132
|
-
TABLE_NAME: 'MT_MAIN',
|
|
133
|
-
LIMITER_RESULT: path => `"${encodeURIComponent(path)}_LIMITER_RESULT"`,
|
|
134
|
-
LIMITER_DATA: path => `"${encodeURIComponent(path)}_LIMITER_DATA"`,
|
|
135
|
-
DB_COUNT_QUERY: path => `"${encodeURIComponent(path)}_DB_COUNT_QUERY"`,
|
|
136
|
-
FETCH_RESOURCES: projectUrl => `FETCH_RESOURCES_${encodeURIComponent(projectUrl)}.db`
|
|
137
|
-
};
|
|
138
|
-
|
|
139
|
-
export const SQLITE_COMMANDS = {
|
|
140
|
-
MERGE: (table, columns = []) => `INSERT OR REPLACE INTO ${table} (${columns.join(', ')}) VALUES (${columns.fill('?').join(', ')})`,
|
|
141
|
-
UPDATE_COLUMNS: (table, columns = [], query = '') => `UPDATE ${table} SET ${columns.map(v => `${v} = ?`).join(', ')} WHERE ${query}`,
|
|
142
|
-
CREATE_INDEX: (table, columns) => `CREATE INDEX idx_${columns.join('_')} ON ${table}(${columns.join(', ')})`,
|
|
143
|
-
DELETE_ROW: (table, query) => `DELETE FROM ${table} WHERE ${query}`
|
|
144
|
-
}
|