react-native-mosquito-transport 0.0.21 → 0.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -7
- package/TODO +18 -8
- package/ios/Mosquitodb.swift +0 -4
- package/package.json +7 -7
- package/src/helpers/engine_api.js +2 -7
- package/src/helpers/peripherals.js +7 -31
- package/src/helpers/purger.js +264 -0
- package/src/helpers/sqlite_manager.js +138 -0
- package/src/helpers/utils.js +63 -43
- package/src/helpers/values.js +5 -20
- package/src/helpers/variables.js +36 -10
- package/src/index.d.ts +97 -56
- package/src/index.js +42 -38
- package/src/products/auth/accessor.js +10 -11
- package/src/products/auth/index.js +13 -28
- package/src/products/database/accessor.js +429 -165
- package/src/products/database/counter.js +12 -10
- package/src/products/database/index.js +177 -162
- package/src/products/database/types.js +1 -0
- package/src/products/database/validator.js +1 -1
- package/src/products/http_callable/accessor.js +72 -0
- package/src/products/http_callable/counter.js +11 -0
- package/src/products/http_callable/index.js +57 -66
- package/src/products/storage/index.js +20 -13
package/README.md
CHANGED
|
@@ -19,14 +19,11 @@ yarn add react-native-mosquito-transport
|
|
|
19
19
|
```js
|
|
20
20
|
import RNMosquitoTransport from "react-native-mosquito-transport";
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
cacheProtocol: "sqlite",
|
|
25
|
-
});
|
|
22
|
+
// uses sqlite to cache it data by default
|
|
23
|
+
RNMosquitoTransport.initializeCache();
|
|
26
24
|
|
|
27
25
|
const mclient = new RNMosquitoTransport({
|
|
28
|
-
projectUrl: "http://localhost:3444"
|
|
29
|
-
accessKey: "SERVER_ACCESS_KEY",
|
|
26
|
+
projectUrl: "http://localhost:3444"
|
|
30
27
|
});
|
|
31
28
|
```
|
|
32
29
|
|
|
@@ -37,7 +34,6 @@ const mclient = new RNMosquitoTransport({
|
|
|
37
34
|
- [dbUrl](#dbUrl)
|
|
38
35
|
- [projectUrl](#projectUrl)
|
|
39
36
|
- [disableCache](#disableCache)
|
|
40
|
-
- [accessKey](#accessKey)
|
|
41
37
|
- [maxRetries](#maxRetries)
|
|
42
38
|
- [enableE2E_Encryption](#enableE2E_Encryption)
|
|
43
39
|
- [serverE2E_PublicKey](#serverE2E_PublicKey)
|
package/TODO
CHANGED
|
@@ -1,13 +1,23 @@
|
|
|
1
|
-
- fix local cache query on sqlite
|
|
1
|
+
- fix local cache query on sqlite ✅
|
|
2
2
|
- fix and add all mongodb query and update operator
|
|
3
3
|
- reauthenticate
|
|
4
4
|
- change `Object` in d.ts to [key: string]: any
|
|
5
5
|
- add `getServerTimeOffset` method
|
|
6
|
-
- change
|
|
7
|
-
- `provide functionality to add extra header to MT instance (all outgoing request)`
|
|
6
|
+
- change undefined to null in `value` ✅
|
|
7
|
+
- `provide functionality to add extra header to MT instance (all outgoing request)` ✅
|
|
8
8
|
- borrowToken
|
|
9
|
-
- `add sqlite`
|
|
10
|
-
- minimize extraction data
|
|
11
|
-
- change `collection().onDisconnect()` to `collection().socket().onDisconnect()` and `collection().socket().onConnect()`
|
|
12
|
-
- add `_foreign_doc` to d.ts
|
|
13
|
-
- tree shake dependencies
|
|
9
|
+
- `add sqlite` ✅
|
|
10
|
+
- minimize extraction data ✅
|
|
11
|
+
- change `collection().onDisconnect()` to `collection().socket().onDisconnect()` and `collection().socket().onConnect()` ✅
|
|
12
|
+
- add `_foreign_doc` to d.ts ✅
|
|
13
|
+
- tree shake dependencies
|
|
14
|
+
- dynamic import for fs ✅
|
|
15
|
+
- new URL() work around ✅
|
|
16
|
+
- fetchHttp, default retrieval if has body ✅
|
|
17
|
+
- native hashing
|
|
18
|
+
- TextEncoder
|
|
19
|
+
- native storage upload
|
|
20
|
+
- lodashes ✅
|
|
21
|
+
- switch to events package
|
|
22
|
+
- serverTimeOffset
|
|
23
|
+
<!-- - error: "refreshToken retry limit exceeded" <--- no need -->
|
package/ios/Mosquitodb.swift
CHANGED
|
@@ -88,7 +88,6 @@ class MosquitodbUploadTask: NSObject, URLSessionDataDelegate {
|
|
|
88
88
|
let filepath = options["file"] as! String
|
|
89
89
|
let url = options["url"] as! String
|
|
90
90
|
let destination = options["destination"] as! String
|
|
91
|
-
let authorization = options["authorization"] as! String
|
|
92
91
|
|
|
93
92
|
do {
|
|
94
93
|
let rawData = try Data(contentsOf: URL(fileURLWithPath: filepath))
|
|
@@ -102,7 +101,6 @@ class MosquitodbUploadTask: NSObject, URLSessionDataDelegate {
|
|
|
102
101
|
}
|
|
103
102
|
}
|
|
104
103
|
request.setValue("application/json", forHTTPHeaderField: "Accept")
|
|
105
|
-
request.setValue(authorization, forHTTPHeaderField: "Authorization")
|
|
106
104
|
if options["authToken"] != nil {
|
|
107
105
|
request.setValue(options["authToken"] as? String, forHTTPHeaderField: "Mosquito-Token")
|
|
108
106
|
}
|
|
@@ -190,7 +188,6 @@ class MosquitodbDownloadTask: NSObject, URLSessionDownloadDelegate {
|
|
|
190
188
|
func downloadFile(options: [String: Any], completion: @escaping ([Any]?)->()) -> Void {
|
|
191
189
|
let processID = options["processID"] as! String
|
|
192
190
|
let url = options["url"] as! String
|
|
193
|
-
let authorization = options["authorization"] as! String
|
|
194
191
|
|
|
195
192
|
var request = URLRequest(url: URL(string: url)!)
|
|
196
193
|
request.httpMethod = "POST"
|
|
@@ -200,7 +197,6 @@ class MosquitodbDownloadTask: NSObject, URLSessionDownloadDelegate {
|
|
|
200
197
|
request.setValue(value, forHTTPHeaderField: key)
|
|
201
198
|
}
|
|
202
199
|
}
|
|
203
|
-
request.setValue(authorization, forHTTPHeaderField: "Authorization")
|
|
204
200
|
if options["authToken"] != nil {
|
|
205
201
|
request.setValue(options["authToken"] as? String, forHTTPHeaderField: "Mosquito-Token")
|
|
206
202
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "react-native-mosquito-transport",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.22",
|
|
4
4
|
"description": "React native javascript sdk for mosquito-transport (https://github.com/brainbehindx/mosquito-transport)",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -30,14 +30,11 @@
|
|
|
30
30
|
"@turf/turf": "^7.1.0",
|
|
31
31
|
"bson": "^6.8.0",
|
|
32
32
|
"buffer": "^6.0.3",
|
|
33
|
-
"crypto-js": "^4.2.0",
|
|
34
33
|
"entity-serializer": "^1.0.2",
|
|
35
|
-
"guard-object": "^1.1.
|
|
36
|
-
"lodash
|
|
37
|
-
"lodash.get": "^4.4.2",
|
|
38
|
-
"lodash.set": "^4.3.2",
|
|
39
|
-
"lodash.unset": "^4.5.2",
|
|
34
|
+
"guard-object": "^1.1.4",
|
|
35
|
+
"lodash": "^4.17.21",
|
|
40
36
|
"react-native-get-random-values": "^1.9.0",
|
|
37
|
+
"react-native-hash": "^3.0.3",
|
|
41
38
|
"simplify-error": "^1.0.1",
|
|
42
39
|
"socket.io-client": "^4.6.2",
|
|
43
40
|
"subscription-listener": "^1.1.2",
|
|
@@ -46,5 +43,8 @@
|
|
|
46
43
|
"peerDependencies": {
|
|
47
44
|
"react": "*",
|
|
48
45
|
"react-native": "*"
|
|
46
|
+
},
|
|
47
|
+
"devDependencies": {
|
|
48
|
+
"@types/react-native-sqlite-storage": "^6.0.5"
|
|
49
49
|
}
|
|
50
50
|
}
|
|
@@ -1,12 +1,7 @@
|
|
|
1
1
|
import { encodeBinary } from './peripherals';
|
|
2
2
|
|
|
3
|
-
const EngineApiBase = (baseApi, ugly, path) =>
|
|
4
|
-
|
|
5
|
-
if (ugly) {
|
|
6
|
-
url.pathname = `/e2e/${encodeBinary(path)}`;
|
|
7
|
-
} else url.pathname = path;
|
|
8
|
-
return url.href;
|
|
9
|
-
};
|
|
3
|
+
const EngineApiBase = (baseApi, ugly, path) =>
|
|
4
|
+
ugly ? `${baseApi}/e2e/${encodeBinary(path)}` : `${baseApi}/${path}`;
|
|
10
5
|
|
|
11
6
|
const apis = {
|
|
12
7
|
_readDocument: (baseApi, ugly) => EngineApiBase(baseApi, ugly, '_readDocument'),
|
|
@@ -1,12 +1,10 @@
|
|
|
1
1
|
import { Buffer } from "buffer";
|
|
2
2
|
import { ServerReachableListener } from "./listeners";
|
|
3
|
-
import aes_pkg from 'crypto-js/aes.js';
|
|
4
|
-
import Utf8Encoder from 'crypto-js/enc-utf8.js';
|
|
5
3
|
import naclPkg from 'tweetnacl';
|
|
6
|
-
import getLodash from "lodash
|
|
4
|
+
import getLodash from "lodash/get";
|
|
7
5
|
import { deserialize, serialize } from "entity-serializer";
|
|
6
|
+
import { CONSTANTS, JSHash } from 'react-native-hash';
|
|
8
7
|
|
|
9
|
-
const { encrypt, decrypt } = aes_pkg;
|
|
10
8
|
const { box, randomBytes } = naclPkg;
|
|
11
9
|
|
|
12
10
|
export const listenReachableServer = (callback, projectUrl) => {
|
|
@@ -42,7 +40,7 @@ export const normalizeRoute = (route = '') => route.split('').map((v, i, a) =>
|
|
|
42
40
|
).join('');
|
|
43
41
|
|
|
44
42
|
export const shuffleArray = (n) => {
|
|
45
|
-
const array =
|
|
43
|
+
const array = n.slice(0);
|
|
46
44
|
let currentIndex = array.length, randomIndex;
|
|
47
45
|
|
|
48
46
|
while (currentIndex != 0) {
|
|
@@ -66,24 +64,10 @@ export function sortArrayByObjectKey(arr = [], key) {
|
|
|
66
64
|
});
|
|
67
65
|
};
|
|
68
66
|
|
|
69
|
-
export async function niceHash(str) {
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
const data = encoder.encode(str);
|
|
74
|
-
|
|
75
|
-
// Use the Web Crypto API to compute the hash
|
|
76
|
-
const hashBuffer = await crypto.subtle.digest('SHA-256', data);
|
|
77
|
-
|
|
78
|
-
// Convert the ArrayBuffer to a hex string for readability
|
|
79
|
-
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
|
80
|
-
const hashHex = hashArray.map(byte => byte.toString(16).padStart(2, '0')).join('');
|
|
81
|
-
|
|
82
|
-
// Convert to base64
|
|
83
|
-
return Buffer.from(hashHex, 'hex').toString('base64');
|
|
84
|
-
} catch (_) {
|
|
85
|
-
return str;
|
|
86
|
-
}
|
|
67
|
+
export async function niceHash(str = '') {
|
|
68
|
+
const hash = await JSHash(str, CONSTANTS.HashAlgorithms.md5);
|
|
69
|
+
if (hash.length > str.length) return encodeBinary(str);
|
|
70
|
+
return hash;
|
|
87
71
|
};
|
|
88
72
|
|
|
89
73
|
export const sameInstance = (var1, var2) => {
|
|
@@ -95,14 +79,6 @@ export const sameInstance = (var1, var2) => {
|
|
|
95
79
|
}
|
|
96
80
|
};
|
|
97
81
|
|
|
98
|
-
export const encryptString = (txt, password, iv) => {
|
|
99
|
-
return encrypt(txt, `${password || ''}${iv || ''}`).toString();
|
|
100
|
-
};
|
|
101
|
-
|
|
102
|
-
export const decryptString = (txt, password, iv) => {
|
|
103
|
-
return decrypt(txt, `${password || ''}${iv || ''}`).toString(Utf8Encoder);
|
|
104
|
-
};
|
|
105
|
-
|
|
106
82
|
export const serializeE2E = async (data, auth_token, serverPublicKey) => {
|
|
107
83
|
const pair = box.keyPair(),
|
|
108
84
|
nonce = randomBytes(box.nonceLength);
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
import { openDB, SQLITE_COMMANDS, SQLITE_PATH } from "./sqlite_manager";
|
|
2
|
+
import { Validator } from "guard-object";
|
|
3
|
+
import { incrementDatabaseSizeCore } from "../products/database/counter";
|
|
4
|
+
import { incrementFetcherSizeCore } from "../products/http_callable/counter";
|
|
5
|
+
import unsetLodash from 'lodash/unset';
|
|
6
|
+
|
|
7
|
+
const { LIMITER_DATA, LIMITER_RESULT, DB_COUNT_QUERY, FETCH_RESOURCES } = SQLITE_PATH;
|
|
8
|
+
|
|
9
|
+
export const purgeRedundantRecords = async (data, builder) => {
|
|
10
|
+
const { io, maxLocalDatabaseSize = 10485760, maxLocalFetchHttpSize = 10485760 } = builder;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* @type {import('./variables')['CacheStore']['DatabaseStats']}
|
|
14
|
+
*/
|
|
15
|
+
const { _db_size, _fetcher_size, counters, database, fetchers } = data.DatabaseStats || {};
|
|
16
|
+
|
|
17
|
+
if (io) {
|
|
18
|
+
const purgeDatabase = () => {
|
|
19
|
+
if (!Validator.POSITIVE_NUMBER(_db_size) || !maxLocalDatabaseSize || _db_size < maxLocalDatabaseSize) return;
|
|
20
|
+
const DbListing = [];
|
|
21
|
+
|
|
22
|
+
breakDbMap(data.DatabaseStore, (projectUrl, dbUrl, dbName, path, value) => {
|
|
23
|
+
Object.entries(value.instance).forEach(([access_id, obj]) => {
|
|
24
|
+
DbListing.push({
|
|
25
|
+
builder: { projectUrl, dbUrl, dbName },
|
|
26
|
+
path,
|
|
27
|
+
access_id,
|
|
28
|
+
value: obj
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
Object.entries(value.episode).forEach(([access_id, limitObj]) => {
|
|
32
|
+
Object.entries(limitObj).forEach(([limit, obj]) => {
|
|
33
|
+
DbListing.push({
|
|
34
|
+
builder: { projectUrl, dbUrl, dbName },
|
|
35
|
+
path,
|
|
36
|
+
access_id,
|
|
37
|
+
limit,
|
|
38
|
+
value: obj,
|
|
39
|
+
isEpisode: true
|
|
40
|
+
});
|
|
41
|
+
});
|
|
42
|
+
});
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
breakDbMap(data.DatabaseCountResult, (projectUrl, dbUrl, dbName, path, value) => {
|
|
46
|
+
Object.entries(value).forEach(([access_id, obj]) => {
|
|
47
|
+
DbListing.push({
|
|
48
|
+
builder: { projectUrl, dbUrl, dbName },
|
|
49
|
+
path,
|
|
50
|
+
access_id,
|
|
51
|
+
value: obj,
|
|
52
|
+
isCount: true
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
const redundantDbRanking = DbListing.sort((a, b) =>
|
|
58
|
+
a.value.touched - b.value.touched
|
|
59
|
+
);
|
|
60
|
+
|
|
61
|
+
const newSize = maxLocalDatabaseSize / 2;
|
|
62
|
+
let sizer = _db_size;
|
|
63
|
+
let cuts = 0;
|
|
64
|
+
|
|
65
|
+
for (let i = 0; i < redundantDbRanking.length; i++) {
|
|
66
|
+
sizer -= redundantDbRanking[i].value.size || 0;
|
|
67
|
+
++cuts;
|
|
68
|
+
if (sizer < newSize) break;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
console.warn(`purging ${cuts} of ${redundantDbRanking.length} db entities`);
|
|
72
|
+
redundantDbRanking.slice(0, cuts).forEach(({
|
|
73
|
+
builder,
|
|
74
|
+
path,
|
|
75
|
+
access_id,
|
|
76
|
+
isCount,
|
|
77
|
+
isEpisode,
|
|
78
|
+
limit,
|
|
79
|
+
value: { size }
|
|
80
|
+
}) => {
|
|
81
|
+
const { projectUrl, dbUrl, dbName } = builder;
|
|
82
|
+
if (isCount) {
|
|
83
|
+
unsetLodash(data.DatabaseCountResult, [projectUrl, dbUrl, dbName, path, access_id]);
|
|
84
|
+
} else {
|
|
85
|
+
incrementDatabaseSizeCore(data.DatabaseStats, builder, path, -size);
|
|
86
|
+
if (isEpisode) {
|
|
87
|
+
unsetLodash(data.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'episode', access_id, limit]);
|
|
88
|
+
} else {
|
|
89
|
+
unsetLodash(data.DatabaseStore, [projectUrl, dbUrl, dbName, path, 'instance', access_id]);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
const purgeFetcher = () => {
|
|
95
|
+
if (!Validator.POSITIVE_NUMBER(_fetcher_size) || !maxLocalFetchHttpSize || _fetcher_size < maxLocalFetchHttpSize) return;
|
|
96
|
+
const redundantFetchRanking = Object.entries(data.FetchedStore).map(([projectUrl, access_id_Obj]) =>
|
|
97
|
+
Object.entries(access_id_Obj).map(([access_id, data]) => ({
|
|
98
|
+
access_id,
|
|
99
|
+
projectUrl,
|
|
100
|
+
data
|
|
101
|
+
}))
|
|
102
|
+
).flat().sort(([a], [b]) =>
|
|
103
|
+
a.data.touched - b.data.touched
|
|
104
|
+
);
|
|
105
|
+
|
|
106
|
+
const newSize = maxLocalFetchHttpSize / 2;
|
|
107
|
+
let sizer = _fetcher_size;
|
|
108
|
+
let cuts = 0;
|
|
109
|
+
|
|
110
|
+
for (let i = 0; i < redundantFetchRanking.length; i++) {
|
|
111
|
+
sizer -= redundantFetchRanking[i].data.size || 0;
|
|
112
|
+
++cuts;
|
|
113
|
+
if (sizer < newSize) break;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
console.warn(`purging ${cuts} of ${redundantFetchRanking.length} fetcher entities`);
|
|
117
|
+
redundantFetchRanking.slice(0, cuts).forEach(({ access_id, data: { size }, projectUrl }) => {
|
|
118
|
+
incrementFetcherSizeCore(data.DatabaseStats, projectUrl, -size);
|
|
119
|
+
unsetLodash(data.FetchedStore, [projectUrl, access_id]);
|
|
120
|
+
});
|
|
121
|
+
console.log('fetcher purging complete');
|
|
122
|
+
}
|
|
123
|
+
purgeDatabase();
|
|
124
|
+
purgeFetcher();
|
|
125
|
+
} else {
|
|
126
|
+
// purge redundant data
|
|
127
|
+
await Promise.allSettled([
|
|
128
|
+
(async () => {
|
|
129
|
+
try {
|
|
130
|
+
if (!Validator.POSITIVE_NUMBER(_db_size) || !maxLocalDatabaseSize || _db_size < maxLocalDatabaseSize) return;
|
|
131
|
+
const instances = [];
|
|
132
|
+
|
|
133
|
+
[database, counters].forEach((map, i) => {
|
|
134
|
+
breakDbMap(map, (projectUrl, dbUrl, dbName, path) => {
|
|
135
|
+
instances.push({
|
|
136
|
+
builder: { projectUrl, dbUrl, dbName },
|
|
137
|
+
isCounter: !!i,
|
|
138
|
+
path
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
const redundantDbRanking = await Promise.all(
|
|
144
|
+
instances.map(async obj => {
|
|
145
|
+
const { builder, isCounter, path } = obj;
|
|
146
|
+
|
|
147
|
+
const sqlite = await openDB(builder);
|
|
148
|
+
|
|
149
|
+
try {
|
|
150
|
+
if (isCounter) {
|
|
151
|
+
const data = await sqlite.executeSql(`SELECT * FROM ${DB_COUNT_QUERY(path)}`).then(r =>
|
|
152
|
+
r[0].rows.raw()
|
|
153
|
+
);
|
|
154
|
+
return data.map(v => [v, obj]);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const [instanceData, resultData] = await Promise.all([
|
|
158
|
+
sqlite.executeSql(`SELECT access_id, touched, size FROM ${LIMITER_DATA(path)}`),
|
|
159
|
+
sqlite.executeSql(`SELECT access_id-limit, touched, size FROM ${LIMITER_RESULT(path)}`)
|
|
160
|
+
]).then(r =>
|
|
161
|
+
r.map(v => v[0].rows.raw())
|
|
162
|
+
);
|
|
163
|
+
return [...instanceData, ...resultData].map(v => [v, obj]);
|
|
164
|
+
} catch (error) {
|
|
165
|
+
console.error('redundantDbRanking err:', error);
|
|
166
|
+
return [];
|
|
167
|
+
} finally {
|
|
168
|
+
sqlite.close();
|
|
169
|
+
}
|
|
170
|
+
})
|
|
171
|
+
).then(r =>
|
|
172
|
+
r.flat().sort(([a], [b]) =>
|
|
173
|
+
a.touched - b.touched
|
|
174
|
+
)
|
|
175
|
+
);
|
|
176
|
+
const newSize = maxLocalDatabaseSize / 2;
|
|
177
|
+
let sizer = _db_size;
|
|
178
|
+
let cuts = 0;
|
|
179
|
+
|
|
180
|
+
for (let i = 0; i < redundantDbRanking.length; i++) {
|
|
181
|
+
sizer -= redundantDbRanking[i][0].size || 0;
|
|
182
|
+
++cuts;
|
|
183
|
+
if (sizer < newSize) break;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
console.warn(`purging ${cuts} of ${redundantDbRanking.length} db entities`);
|
|
187
|
+
await Promise.all(redundantDbRanking.slice(0, cuts).map(async ([v, { builder, isCounter, path }]) => {
|
|
188
|
+
const sqlite = await openDB(builder);
|
|
189
|
+
try {
|
|
190
|
+
const table = (isCounter ? DB_COUNT_QUERY : 'access_id-limit' in v ? LIMITER_RESULT : LIMITER_DATA)(path);
|
|
191
|
+
const id_field = 'access_id-limit' in v ? 'access_id-limit' : 'access_id';
|
|
192
|
+
|
|
193
|
+
await sqlite.executeSql(SQLITE_COMMANDS.DELETE_ROW(table, `${id_field} = ?`), [v[id_field]]);
|
|
194
|
+
if (!isCounter) incrementDatabaseSizeCore(data.DatabaseStats, builder, path, -v.size);
|
|
195
|
+
} catch (error) {
|
|
196
|
+
console.log('db redundantClearing err:', error);
|
|
197
|
+
}
|
|
198
|
+
sqlite.close();
|
|
199
|
+
}));
|
|
200
|
+
console.log('database purging complete');
|
|
201
|
+
} catch (error) {
|
|
202
|
+
console.error('database purging err:', error);
|
|
203
|
+
}
|
|
204
|
+
})(),
|
|
205
|
+
(async () => {
|
|
206
|
+
try {
|
|
207
|
+
if (!Validator.POSITIVE_NUMBER(_fetcher_size) || !maxLocalFetchHttpSize || _fetcher_size < maxLocalFetchHttpSize) return;
|
|
208
|
+
|
|
209
|
+
const redundantFetchRanking = await Promise.all(Object.entries(fetchers).map(async ([projectUrl]) => {
|
|
210
|
+
const sqlite = await openDB(FETCH_RESOURCES(projectUrl));
|
|
211
|
+
const data = await sqlite.executeSql('SELECT access_id, touched, size FROM main').then(r =>
|
|
212
|
+
r[0].rows.raw()
|
|
213
|
+
);
|
|
214
|
+
return data.map(v => [v, projectUrl]);
|
|
215
|
+
})).then(r =>
|
|
216
|
+
r.flat().sort(([a], [b]) =>
|
|
217
|
+
a.touched - b.touched
|
|
218
|
+
)
|
|
219
|
+
);
|
|
220
|
+
|
|
221
|
+
const newSize = maxLocalFetchHttpSize / 2;
|
|
222
|
+
let sizer = _fetcher_size;
|
|
223
|
+
let cuts = 0;
|
|
224
|
+
|
|
225
|
+
for (let i = 0; i < redundantFetchRanking.length; i++) {
|
|
226
|
+
sizer -= redundantFetchRanking[i][0].size || 0;
|
|
227
|
+
++cuts;
|
|
228
|
+
if (sizer < newSize) break;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
console.warn(`purging ${cuts} of ${redundantFetchRanking.length} fetcher entities`);
|
|
232
|
+
await Promise.all(redundantFetchRanking.slice(0, cuts).map(async ([v, projectUrl]) => {
|
|
233
|
+
const sqlite = await openDB(FETCH_RESOURCES(projectUrl));
|
|
234
|
+
try {
|
|
235
|
+
await sqlite.executeSql(SQLITE_COMMANDS.DELETE_ROW('main', 'access_id = ?'), [v.access_id]);
|
|
236
|
+
incrementFetcherSizeCore(data.DatabaseStats, projectUrl, -v.size);
|
|
237
|
+
} catch (error) {
|
|
238
|
+
console.log('fetcher redundantClearing err:', error);
|
|
239
|
+
}
|
|
240
|
+
sqlite.close();
|
|
241
|
+
}));
|
|
242
|
+
console.log('fetcher purging complete');
|
|
243
|
+
} catch (error) {
|
|
244
|
+
console.error('fetcher purging err:', error);
|
|
245
|
+
}
|
|
246
|
+
})()
|
|
247
|
+
]);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
const breakDbMap = (obj, callback) =>
|
|
252
|
+
Object.entries(obj).forEach(([projectUrl, dbUrlObj]) => {
|
|
253
|
+
Object.entries(dbUrlObj).forEach(([dbUrl, dbNameObj]) => {
|
|
254
|
+
Object.entries(dbNameObj).forEach(([dbName, pathObj]) => {
|
|
255
|
+
Object.entries(pathObj).forEach(([path, value]) => {
|
|
256
|
+
callback(projectUrl, dbUrl, dbName, path, value);
|
|
257
|
+
});
|
|
258
|
+
});
|
|
259
|
+
});
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
export const purgeInstance = (projectUrl) => {
|
|
263
|
+
// TODO: purge when signed-out
|
|
264
|
+
}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { enablePromise, openDatabase } from 'react-native-sqlite-storage';
|
|
2
|
+
import { Scoped, SqliteCollective } from './variables';
|
|
3
|
+
import { niceHash } from './peripherals';
|
|
4
|
+
|
|
5
|
+
enablePromise(true);
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* this method implement a centralize approach for opening and closing of sqlite database to ensure consistency across multiple task opening and closing the database in diferent order
|
|
9
|
+
*
|
|
10
|
+
* @param {string} name
|
|
11
|
+
* @returns {Promise<import('react-native-sqlite-storage').SQLiteDatabase>}
|
|
12
|
+
*/
|
|
13
|
+
export const openDB = async (name) => {
|
|
14
|
+
|
|
15
|
+
if (name?.projectUrl) {
|
|
16
|
+
const { projectUrl, dbUrl, dbName } = name;
|
|
17
|
+
name = encodeURIComponent(`${projectUrl}_${dbUrl}_${dbName}`) + '.db';
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const { sqliteKey } = Scoped.ReleaseCacheData;
|
|
21
|
+
|
|
22
|
+
if (sqliteKey) name = `${niceHash(sqliteKey)}__${name}`;
|
|
23
|
+
|
|
24
|
+
if (!SqliteCollective.openedDb[name]) {
|
|
25
|
+
SqliteCollective.openedDbProcess[name] = 0;
|
|
26
|
+
SqliteCollective.openedDb[name] = (SqliteCollective.closeDbPromises[name] || Promise.resolve()).finally(() =>
|
|
27
|
+
openDatabase({
|
|
28
|
+
location: 'default',
|
|
29
|
+
name,
|
|
30
|
+
key: sqliteKey
|
|
31
|
+
}).then(db => {
|
|
32
|
+
const prevClose = db.close.bind(db);
|
|
33
|
+
|
|
34
|
+
db.close = () => new Promise((resolve, reject) => {
|
|
35
|
+
if (--SqliteCollective.openedDbProcess[name] === 0) {
|
|
36
|
+
let willClose;
|
|
37
|
+
const timer = setTimeout(async () => {
|
|
38
|
+
willClose = true;
|
|
39
|
+
delete SqliteCollective.openedDb[name];
|
|
40
|
+
delete SqliteCollective.openedDbProcess[name];
|
|
41
|
+
SqliteCollective.closeDbPromises[name] = prevClose().then(() => {
|
|
42
|
+
resolve('active');
|
|
43
|
+
}).catch(e => {
|
|
44
|
+
reject(new Error(`${e}`));
|
|
45
|
+
}).finally(() => {
|
|
46
|
+
delete SqliteCollective.closeDbPromises[name];
|
|
47
|
+
});
|
|
48
|
+
delete SqliteCollective.openedDbReducerTimer[name];
|
|
49
|
+
}, 7);
|
|
50
|
+
|
|
51
|
+
SqliteCollective.openedDbReducerTimer[name] = () => {
|
|
52
|
+
clearTimeout(timer);
|
|
53
|
+
if (!willClose) resolve('passive');
|
|
54
|
+
delete SqliteCollective.openedDbReducerTimer[name];
|
|
55
|
+
}
|
|
56
|
+
} else resolve('passive');
|
|
57
|
+
});
|
|
58
|
+
return db;
|
|
59
|
+
})
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
SqliteCollective.openedDbReducerTimer[name]?.();
|
|
64
|
+
++SqliteCollective.openedDbProcess[name];
|
|
65
|
+
const thisDb = await SqliteCollective.openedDb[name];
|
|
66
|
+
let hasClosed;
|
|
67
|
+
|
|
68
|
+
const thisClose = async () => {
|
|
69
|
+
if (hasClosed) return;
|
|
70
|
+
hasClosed = true;
|
|
71
|
+
return (await thisDb.close());
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return new Proxy({}, {
|
|
75
|
+
get: (_, n) => {
|
|
76
|
+
if (n === 'close') {
|
|
77
|
+
return thisClose;
|
|
78
|
+
} else if (typeof thisDb[n] === 'function')
|
|
79
|
+
return thisDb[n].bind(thisDb);
|
|
80
|
+
return thisDb[n];
|
|
81
|
+
},
|
|
82
|
+
set: (_, n, v) => {
|
|
83
|
+
thisDb[n] = v;
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* this method linearize read/write on sqlite ensuring consistency across concurrent operations
|
|
90
|
+
*
|
|
91
|
+
* @param {any} builder
|
|
92
|
+
* @param {string} access_id
|
|
93
|
+
* @param {'database' | 'dbQueryCount' | 'httpFetch'} node
|
|
94
|
+
* @returns {(task: (sqlite: import("react-native-sqlite-storage").SQLiteDatabase) => Promise<{any}> )=> Promise<{any}>}
|
|
95
|
+
*/
|
|
96
|
+
export const useSqliteLinearAccessId = (builder, access_id, node) => async (task) => {
|
|
97
|
+
const { projectUrl, dbUrl, dbName } = builder;
|
|
98
|
+
const nodeId = typeof builder === 'string' ? `${builder}_${access_id}` : `${projectUrl}_${dbUrl}_${dbName}_${access_id}`;
|
|
99
|
+
|
|
100
|
+
const sqlite = await openDB(builder);
|
|
101
|
+
|
|
102
|
+
const thatProcess = Scoped.linearSqliteProcess[node][nodeId];
|
|
103
|
+
|
|
104
|
+
const thisPromise = new Promise(async (resolve, reject) => {
|
|
105
|
+
try {
|
|
106
|
+
if (thatProcess !== undefined) await thatProcess;
|
|
107
|
+
} catch (_) { }
|
|
108
|
+
try {
|
|
109
|
+
resolve(await task(sqlite));
|
|
110
|
+
} catch (error) {
|
|
111
|
+
console.error('useSqliteLinearAccessId err:', error);
|
|
112
|
+
reject(error);
|
|
113
|
+
} finally {
|
|
114
|
+
if (Scoped.linearSqliteProcess[node][nodeId] === thisPromise)
|
|
115
|
+
delete Scoped.linearSqliteProcess[node][nodeId];
|
|
116
|
+
sqlite.close();
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
Scoped.linearSqliteProcess[node][nodeId] = thisPromise;
|
|
121
|
+
return (await thisPromise);
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
export const SQLITE_PATH = {
|
|
125
|
+
FILE_NAME: 'MOSQUITO_TRANSPORT.db',
|
|
126
|
+
TABLE_NAME: 'MT_MAIN',
|
|
127
|
+
LIMITER_RESULT: path => `"${encodeURIComponent(path)}-LIMITER_RESULT"`,
|
|
128
|
+
LIMITER_DATA: path => `"${encodeURIComponent(path)}-LIMITER_DATA"`,
|
|
129
|
+
DB_COUNT_QUERY: path => `"${encodeURIComponent(path)}-DB_COUNT_QUERY"`,
|
|
130
|
+
FETCH_RESOURCES: projectUrl => `FETCH_RESOURCES-${encodeURIComponent(projectUrl)}.db`
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
export const SQLITE_COMMANDS = {
|
|
134
|
+
MERGE: (table, columns = []) => `INSERT INTO ${table} (${columns.join(', ')}) VALUES (${columns.map(() => '?').join(', ')}) ON CONFLICT(${columns[0]}) DO UPDATE SET ${columns.slice(1).map(v => `${v} = excluded.${v}`).join(', ')}`,
|
|
135
|
+
UPDATE_COLUMNS: (table, columns = [], query = '') => `UPDATE ${table} SET ${columns.map(v => `${v} = ?`).join(', ')} WHERE ${query}`,
|
|
136
|
+
CREATE_INDEX: (table, columns) => `CREATE INDEX idx_${columns.join('_')} ON ${table}(${columns.join(', ')})`,
|
|
137
|
+
DELETE_ROW: (table, query) => `DELETE FROM ${table} WHERE ${query}`
|
|
138
|
+
}
|