react-native-nitro-storage 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +107 -15
- package/SECURITY.md +2 -2
- package/android/build.gradle +5 -0
- package/android/src/main/cpp/AndroidStorageAdapterCpp.cpp +12 -13
- package/android/src/main/cpp/JniSize.hpp +21 -0
- package/android/src/main/java/com/nitrostorage/AndroidStorageAdapter.kt +94 -7
- package/android/src/main/java/com/nitrostorage/DiskSqliteStore.kt +147 -34
- package/cpp/bindings/HybridStorage.cpp +12 -4
- package/cpp/core/SqliteDiskStore.cpp +172 -26
- package/cpp/core/SqliteDiskStore.hpp +10 -1
- package/docs/api-reference.md +60 -54
- package/docs/batch-transactions-migrations.md +7 -0
- package/docs/native-libraries.md +28 -0
- package/docs/qa/agent-device-replay.md +70 -0
- package/docs/secure-storage.md +12 -6
- package/ios/IOSStorageAdapterCpp.mm +89 -22
- package/lib/commonjs/Storage.types.js +4 -2
- package/lib/commonjs/Storage.types.js.map +1 -1
- package/lib/commonjs/core/durability.js +30 -2
- package/lib/commonjs/core/durability.js.map +1 -1
- package/lib/commonjs/index.js.map +1 -1
- package/lib/commonjs/index.web.js.map +1 -1
- package/lib/commonjs/shared.js.map +1 -1
- package/lib/commonjs/storage-core.js +63 -9
- package/lib/commonjs/storage-core.js.map +1 -1
- package/lib/commonjs/testing.js +1 -0
- package/lib/commonjs/testing.js.map +1 -1
- package/lib/module/Storage.types.js +4 -2
- package/lib/module/Storage.types.js.map +1 -1
- package/lib/module/core/durability.js +30 -2
- package/lib/module/core/durability.js.map +1 -1
- package/lib/module/index.js.map +1 -1
- package/lib/module/index.web.js.map +1 -1
- package/lib/module/shared.js.map +1 -1
- package/lib/module/storage-core.js +63 -9
- package/lib/module/storage-core.js.map +1 -1
- package/lib/module/testing.js +1 -0
- package/lib/module/testing.js.map +1 -1
- package/lib/typescript/Storage.types.d.ts +4 -2
- package/lib/typescript/Storage.types.d.ts.map +1 -1
- package/lib/typescript/core/durability.d.ts +7 -1
- package/lib/typescript/core/durability.d.ts.map +1 -1
- package/lib/typescript/index.d.ts +4 -2
- package/lib/typescript/index.d.ts.map +1 -1
- package/lib/typescript/index.web.d.ts +4 -2
- package/lib/typescript/index.web.d.ts.map +1 -1
- package/lib/typescript/shared.d.ts +8 -1
- package/lib/typescript/shared.d.ts.map +1 -1
- package/lib/typescript/storage-core.d.ts +6 -3
- package/lib/typescript/storage-core.d.ts.map +1 -1
- package/lib/typescript/testing.d.ts +7 -4
- package/lib/typescript/testing.d.ts.map +1 -1
- package/package.json +2 -1
- package/src/Storage.types.ts +4 -2
- package/src/core/durability.ts +46 -2
- package/src/index.ts +4 -0
- package/src/index.web.ts +4 -0
- package/src/shared.ts +17 -1
- package/src/storage-core.ts +87 -13
- package/src/testing.ts +5 -0
|
@@ -2,21 +2,49 @@ package com.nitrostorage
|
|
|
2
2
|
|
|
3
3
|
import android.content.Context
|
|
4
4
|
import android.content.SharedPreferences
|
|
5
|
+
import android.database.DatabaseErrorHandler
|
|
5
6
|
import android.database.sqlite.SQLiteDatabase
|
|
7
|
+
import android.database.sqlite.SQLiteDatabaseCorruptException
|
|
8
|
+
import android.database.sqlite.SQLiteException
|
|
9
|
+
import java.io.ByteArrayOutputStream
|
|
6
10
|
import java.io.File
|
|
7
11
|
|
|
12
|
+
internal typealias DiskDatabaseOpener = (path: String, flags: Int) -> SQLiteDatabase
|
|
13
|
+
|
|
14
|
+
internal fun openDiskDatabase(path: String, flags: Int): SQLiteDatabase {
|
|
15
|
+
return SQLiteDatabase.openDatabase(path, null, flags, DatabaseErrorHandler { })
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
internal fun openDiskDatabaseWithWalFallback(
|
|
19
|
+
path: String,
|
|
20
|
+
opener: DiskDatabaseOpener,
|
|
21
|
+
): SQLiteDatabase {
|
|
22
|
+
val flags = SQLiteDatabase.CREATE_IF_NECESSARY or SQLiteDatabase.NO_LOCALIZED_COLLATORS
|
|
23
|
+
return try {
|
|
24
|
+
opener(path, flags or SQLiteDatabase.ENABLE_WRITE_AHEAD_LOGGING)
|
|
25
|
+
} catch (walError: SQLiteDatabaseCorruptException) {
|
|
26
|
+
throw walError
|
|
27
|
+
} catch (walError: SQLiteException) {
|
|
28
|
+
try {
|
|
29
|
+
opener(path, flags)
|
|
30
|
+
} catch (fallbackError: Exception) {
|
|
31
|
+
walError.addSuppressed(fallbackError)
|
|
32
|
+
throw walError
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
8
37
|
internal class DiskSqliteStore(
|
|
9
38
|
context: Context,
|
|
10
39
|
private val legacyPreferences: SharedPreferences,
|
|
40
|
+
opener: DiskDatabaseOpener = ::openDiskDatabase,
|
|
11
41
|
) {
|
|
12
42
|
private val db: SQLiteDatabase =
|
|
13
|
-
|
|
43
|
+
openDiskDatabaseWithWalFallback(databaseFile(context).path, opener)
|
|
14
44
|
|
|
15
45
|
init {
|
|
16
46
|
try {
|
|
17
|
-
|
|
18
|
-
db.rawQuery("PRAGMA journal_mode=WAL", null).close()
|
|
19
|
-
db.execSQL("PRAGMA synchronous=NORMAL")
|
|
47
|
+
db.execSQL(synchronousPragmaFor(queryPragma("PRAGMA journal_mode")))
|
|
20
48
|
db.execSQL(
|
|
21
49
|
"CREATE TABLE IF NOT EXISTS kv (key TEXT PRIMARY KEY NOT NULL, value TEXT NOT NULL)",
|
|
22
50
|
)
|
|
@@ -30,6 +58,17 @@ internal class DiskSqliteStore(
|
|
|
30
58
|
}
|
|
31
59
|
}
|
|
32
60
|
|
|
61
|
+
private fun queryPragma(sql: String): String? {
|
|
62
|
+
return db.rawQuery(sql, null).use { cursor ->
|
|
63
|
+
if (cursor.moveToFirst()) cursor.getString(0) else null
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
@Synchronized
|
|
68
|
+
fun close() {
|
|
69
|
+
db.close()
|
|
70
|
+
}
|
|
71
|
+
|
|
33
72
|
@Synchronized
|
|
34
73
|
fun set(key: String, value: String) {
|
|
35
74
|
db.execSQL(
|
|
@@ -40,9 +79,39 @@ internal class DiskSqliteStore(
|
|
|
40
79
|
|
|
41
80
|
@Synchronized
|
|
42
81
|
fun get(key: String): String? {
|
|
43
|
-
db.rawQuery(
|
|
44
|
-
|
|
82
|
+
db.rawQuery(
|
|
83
|
+
"SELECT length(CAST(value AS BLOB)), " +
|
|
84
|
+
"CASE WHEN length(CAST(value AS BLOB)) <= $VALUE_CHUNK_BYTES THEN value END " +
|
|
85
|
+
"FROM kv WHERE key = ? LIMIT 1",
|
|
86
|
+
arrayOf(key),
|
|
87
|
+
).use { cursor ->
|
|
88
|
+
if (!cursor.moveToFirst()) {
|
|
89
|
+
return null
|
|
90
|
+
}
|
|
91
|
+
if (!cursor.isNull(1)) {
|
|
92
|
+
return cursor.getString(1)
|
|
93
|
+
}
|
|
94
|
+
return readLargeValue(key, cursor.getLong(0))
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
private fun readLargeValue(key: String, byteCount: Long): String {
|
|
99
|
+
val bytes = ByteArrayOutputStream(byteCount.coerceAtMost(Int.MAX_VALUE.toLong()).toInt())
|
|
100
|
+
var offset = 1L
|
|
101
|
+
while (offset <= byteCount) {
|
|
102
|
+
val chunk = db.rawQuery(
|
|
103
|
+
"SELECT substr(CAST(value AS BLOB), ?, $VALUE_CHUNK_BYTES) FROM kv WHERE key = ?",
|
|
104
|
+
arrayOf(offset.toString(), key),
|
|
105
|
+
).use { cursor ->
|
|
106
|
+
if (cursor.moveToFirst()) cursor.getBlob(0) else null
|
|
107
|
+
}
|
|
108
|
+
if (chunk == null || chunk.isEmpty()) {
|
|
109
|
+
break
|
|
110
|
+
}
|
|
111
|
+
bytes.write(chunk, 0, chunk.size)
|
|
112
|
+
offset += chunk.size
|
|
45
113
|
}
|
|
114
|
+
return String(bytes.toByteArray(), Charsets.UTF_8)
|
|
46
115
|
}
|
|
47
116
|
|
|
48
117
|
@Synchronized
|
|
@@ -61,17 +130,13 @@ internal class DiskSqliteStore(
|
|
|
61
130
|
@Synchronized
|
|
62
131
|
fun setBatch(keys: Array<String>, values: Array<String>) {
|
|
63
132
|
val count = minOf(keys.size, values.size)
|
|
64
|
-
|
|
65
|
-
try {
|
|
133
|
+
inTransaction {
|
|
66
134
|
for (index in 0 until count) {
|
|
67
135
|
db.execSQL(
|
|
68
136
|
"INSERT OR REPLACE INTO kv(key, value) VALUES(?, ?)",
|
|
69
137
|
arrayOf(keys[index], values[index]),
|
|
70
138
|
)
|
|
71
139
|
}
|
|
72
|
-
db.setTransactionSuccessful()
|
|
73
|
-
} finally {
|
|
74
|
-
db.endTransaction()
|
|
75
140
|
}
|
|
76
141
|
}
|
|
77
142
|
|
|
@@ -82,14 +147,10 @@ internal class DiskSqliteStore(
|
|
|
82
147
|
|
|
83
148
|
@Synchronized
|
|
84
149
|
fun removeBatch(keys: Array<String>) {
|
|
85
|
-
|
|
86
|
-
try {
|
|
150
|
+
inTransaction {
|
|
87
151
|
for (key in keys) {
|
|
88
152
|
db.execSQL("DELETE FROM kv WHERE key = ?", arrayOf(key))
|
|
89
153
|
}
|
|
90
|
-
db.setTransactionSuccessful()
|
|
91
|
-
} finally {
|
|
92
|
-
db.endTransaction()
|
|
93
154
|
}
|
|
94
155
|
removeLegacyKeys(keys.asIterable())
|
|
95
156
|
}
|
|
@@ -107,19 +168,12 @@ internal class DiskSqliteStore(
|
|
|
107
168
|
|
|
108
169
|
@Synchronized
|
|
109
170
|
fun getKeysByPrefix(prefix: String): Array<String> {
|
|
110
|
-
val
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
if (character == '%' || character == '_' || character == '\\') append('\\')
|
|
115
|
-
append(character)
|
|
116
|
-
}
|
|
117
|
-
append('%')
|
|
171
|
+
val cursor = if (hasUnpairedSurrogate(prefix)) {
|
|
172
|
+
db.rawQuery("SELECT key FROM kv WHERE key LIKE ? ESCAPE '\\'", arrayOf(likePattern(prefix)))
|
|
173
|
+
} else {
|
|
174
|
+
db.rawQuery("SELECT key FROM kv WHERE key >= ?1 AND key < (?1 || x'FF')", arrayOf(prefix))
|
|
118
175
|
}
|
|
119
|
-
|
|
120
|
-
"SELECT key FROM kv WHERE key LIKE ? ESCAPE '\\'",
|
|
121
|
-
arrayOf(pattern),
|
|
122
|
-
).use { cursor ->
|
|
176
|
+
cursor.use {
|
|
123
177
|
val keys = ArrayList<String>(cursor.count)
|
|
124
178
|
while (cursor.moveToNext()) {
|
|
125
179
|
val key = cursor.getString(0)
|
|
@@ -156,8 +210,7 @@ internal class DiskSqliteStore(
|
|
|
156
210
|
}
|
|
157
211
|
}
|
|
158
212
|
|
|
159
|
-
|
|
160
|
-
try {
|
|
213
|
+
inTransaction {
|
|
161
214
|
for ((key, value) in legacyPreferences.all) {
|
|
162
215
|
if (value is String) {
|
|
163
216
|
db.execSQL(
|
|
@@ -170,9 +223,25 @@ internal class DiskSqliteStore(
|
|
|
170
223
|
"INSERT OR REPLACE INTO meta(k, v) VALUES(?, ?)",
|
|
171
224
|
arrayOf(PREFS_MIGRATION_KEY, "1"),
|
|
172
225
|
)
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
private inline fun inTransaction(block: () -> Unit) {
|
|
230
|
+
db.beginTransaction()
|
|
231
|
+
var failure: Throwable? = null
|
|
232
|
+
try {
|
|
233
|
+
block()
|
|
173
234
|
db.setTransactionSuccessful()
|
|
235
|
+
} catch (error: Throwable) {
|
|
236
|
+
failure = error
|
|
237
|
+
throw error
|
|
174
238
|
} finally {
|
|
175
|
-
|
|
239
|
+
try {
|
|
240
|
+
db.endTransaction()
|
|
241
|
+
} catch (error: SQLiteException) {
|
|
242
|
+
val primary = failure ?: throw error
|
|
243
|
+
primary.addSuppressed(error)
|
|
244
|
+
}
|
|
176
245
|
}
|
|
177
246
|
}
|
|
178
247
|
|
|
@@ -186,8 +255,52 @@ internal class DiskSqliteStore(
|
|
|
186
255
|
editor.apply()
|
|
187
256
|
}
|
|
188
257
|
|
|
189
|
-
|
|
190
|
-
const val DATABASE_NAME = "nitro-storage-disk.sqlite"
|
|
191
|
-
const val PREFS_MIGRATION_KEY = "prefs_v1"
|
|
258
|
+
companion object {
|
|
259
|
+
private const val DATABASE_NAME = "nitro-storage-disk.sqlite"
|
|
260
|
+
private const val PREFS_MIGRATION_KEY = "prefs_v1"
|
|
261
|
+
private const val VALUE_CHUNK_BYTES = 512 * 1024
|
|
262
|
+
|
|
263
|
+
fun databaseFile(context: Context): File = File(context.filesDir, DATABASE_NAME)
|
|
264
|
+
|
|
265
|
+
fun deleteDatabaseFiles(context: Context) {
|
|
266
|
+
SQLiteDatabase.deleteDatabase(databaseFile(context))
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
private fun hasUnpairedSurrogate(value: String): Boolean {
|
|
272
|
+
var index = 0
|
|
273
|
+
while (index < value.length) {
|
|
274
|
+
val character = value[index]
|
|
275
|
+
if (Character.isHighSurrogate(character)) {
|
|
276
|
+
if (index + 1 >= value.length || !Character.isLowSurrogate(value[index + 1])) {
|
|
277
|
+
return true
|
|
278
|
+
}
|
|
279
|
+
index += 2
|
|
280
|
+
} else if (Character.isLowSurrogate(character)) {
|
|
281
|
+
return true
|
|
282
|
+
} else {
|
|
283
|
+
index += 1
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
return false
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
private fun likePattern(prefix: String): String {
|
|
290
|
+
return buildString {
|
|
291
|
+
for (character in prefix) {
|
|
292
|
+
if (character == '\u0000') break
|
|
293
|
+
if (character == '%' || character == '_' || character == '\\') append('\\')
|
|
294
|
+
append(character)
|
|
295
|
+
}
|
|
296
|
+
append('%')
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
internal fun synchronousPragmaFor(journalMode: String?): String {
|
|
301
|
+
return if (journalMode.equals("wal", ignoreCase = true)) {
|
|
302
|
+
"PRAGMA synchronous=NORMAL"
|
|
303
|
+
} else {
|
|
304
|
+
"PRAGMA synchronous=FULL"
|
|
192
305
|
}
|
|
193
306
|
}
|
|
@@ -434,6 +434,7 @@ void HybridStorage::removeBatch(const std::vector<std::string>& keys, double sco
|
|
|
434
434
|
}
|
|
435
435
|
|
|
436
436
|
void HybridStorage::removeByPrefix(const std::string& prefix, double scope) {
|
|
437
|
+
toScope(scope);
|
|
437
438
|
if (prefix.empty()) {
|
|
438
439
|
return;
|
|
439
440
|
}
|
|
@@ -462,17 +463,23 @@ void HybridStorage::setSecureAccessControl(double level) {
|
|
|
462
463
|
throw std::runtime_error("NitroStorage: Invalid access control level");
|
|
463
464
|
}
|
|
464
465
|
ensureAdapter();
|
|
465
|
-
|
|
466
|
+
runAdapterOperation(
|
|
467
|
+
[&] { nativeAdapter_->setSecureAccessControl(intLevel); },
|
|
468
|
+
"Secure access control");
|
|
466
469
|
}
|
|
467
470
|
|
|
468
471
|
void HybridStorage::setSecureWritesAsync(bool enabled) {
|
|
469
472
|
ensureAdapter();
|
|
470
|
-
|
|
473
|
+
runAdapterOperation(
|
|
474
|
+
[&] { nativeAdapter_->setSecureWritesAsync(enabled); },
|
|
475
|
+
"Secure write mode");
|
|
471
476
|
}
|
|
472
477
|
|
|
473
478
|
void HybridStorage::setKeychainAccessGroup(const std::string& group) {
|
|
474
479
|
ensureAdapter();
|
|
475
|
-
|
|
480
|
+
runAdapterOperation(
|
|
481
|
+
[&] { nativeAdapter_->setKeychainAccessGroup(group); },
|
|
482
|
+
"Keychain access group");
|
|
476
483
|
}
|
|
477
484
|
|
|
478
485
|
// --- Biometric ---
|
|
@@ -517,7 +524,8 @@ void HybridStorage::deleteSecureBiometric(const std::string& key) {
|
|
|
517
524
|
|
|
518
525
|
bool HybridStorage::hasSecureBiometric(const std::string& key) {
|
|
519
526
|
ensureAdapter();
|
|
520
|
-
return
|
|
527
|
+
return runAdapterOperation(
|
|
528
|
+
[&] { return nativeAdapter_->hasSecureBiometric(key); }, "Biometric has");
|
|
521
529
|
}
|
|
522
530
|
|
|
523
531
|
void HybridStorage::clearSecureBiometric() {
|
|
@@ -3,10 +3,12 @@
|
|
|
3
3
|
#include <sqlite3.h>
|
|
4
4
|
|
|
5
5
|
#include <algorithm>
|
|
6
|
+
#include <cerrno>
|
|
6
7
|
#include <cstdlib>
|
|
7
8
|
#include <filesystem>
|
|
8
9
|
#include <memory>
|
|
9
10
|
#include <stdexcept>
|
|
11
|
+
#include <system_error>
|
|
10
12
|
|
|
11
13
|
namespace NitroStorage {
|
|
12
14
|
namespace {
|
|
@@ -21,13 +23,19 @@ std::mutex& sharedMutex() {
|
|
|
21
23
|
return mutex;
|
|
22
24
|
}
|
|
23
25
|
|
|
24
|
-
const char* storageErrorTag(int code) {
|
|
26
|
+
const char* storageErrorTag(sqlite3* db, int code) {
|
|
25
27
|
switch (code & 0xff) {
|
|
26
28
|
case SQLITE_FULL:
|
|
27
29
|
return "[nitro-error:storage_full] ";
|
|
28
30
|
case SQLITE_CORRUPT:
|
|
29
31
|
case SQLITE_NOTADB:
|
|
30
32
|
return "[nitro-error:storage_corruption] ";
|
|
33
|
+
case SQLITE_IOERR: {
|
|
34
|
+
const int systemError = db != nullptr ? sqlite3_system_errno(db) : 0;
|
|
35
|
+
return systemError == ENOSPC || systemError == EDQUOT
|
|
36
|
+
? "[nitro-error:storage_full] "
|
|
37
|
+
: "";
|
|
38
|
+
}
|
|
31
39
|
default:
|
|
32
40
|
return "";
|
|
33
41
|
}
|
|
@@ -36,21 +44,22 @@ const char* storageErrorTag(int code) {
|
|
|
36
44
|
[[noreturn]] void throwSqlite(sqlite3* db, const char* operation, int code) {
|
|
37
45
|
const char* message = db != nullptr ? sqlite3_errmsg(db) : sqlite3_errstr(code);
|
|
38
46
|
throw std::runtime_error(
|
|
39
|
-
std::string(storageErrorTag(code)) + "NitroStorage: Disk SQLite " + operation +
|
|
47
|
+
std::string(storageErrorTag(db, code)) + "NitroStorage: Disk SQLite " + operation +
|
|
40
48
|
" failed: " + (message != nullptr ? message : "unknown error")
|
|
41
49
|
);
|
|
42
50
|
}
|
|
43
51
|
|
|
44
52
|
void bindText(sqlite3_stmt* stmt, int index, const std::string& value) {
|
|
45
|
-
const int rc =
|
|
53
|
+
const int rc = sqlite3_bind_text64(
|
|
46
54
|
stmt,
|
|
47
55
|
index,
|
|
48
56
|
value.data(),
|
|
49
|
-
static_cast<
|
|
50
|
-
SQLITE_TRANSIENT
|
|
57
|
+
static_cast<sqlite3_uint64>(value.size()),
|
|
58
|
+
SQLITE_TRANSIENT,
|
|
59
|
+
SQLITE_UTF8
|
|
51
60
|
);
|
|
52
61
|
if (rc != SQLITE_OK) {
|
|
53
|
-
|
|
62
|
+
throwSqlite(sqlite3_db_handle(stmt), "bind", rc);
|
|
54
63
|
}
|
|
55
64
|
}
|
|
56
65
|
|
|
@@ -69,16 +78,23 @@ std::string escapeLikePrefix(const std::string& prefix) {
|
|
|
69
78
|
return escaped;
|
|
70
79
|
}
|
|
71
80
|
|
|
81
|
+
std::optional<std::string> prefixUpperBound(const std::string& prefix) {
|
|
82
|
+
std::string upper = prefix;
|
|
83
|
+
while (!upper.empty() && static_cast<unsigned char>(upper.back()) == 0xFF) {
|
|
84
|
+
upper.pop_back();
|
|
85
|
+
}
|
|
86
|
+
if (upper.empty()) {
|
|
87
|
+
return std::nullopt;
|
|
88
|
+
}
|
|
89
|
+
upper.back() = static_cast<char>(static_cast<unsigned char>(upper.back()) + 1);
|
|
90
|
+
return upper;
|
|
91
|
+
}
|
|
92
|
+
|
|
72
93
|
} // namespace
|
|
73
94
|
|
|
74
95
|
SqliteDiskStore::SqliteDiskStore(std::string path) : path_(std::move(path)) {
|
|
75
96
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
76
|
-
|
|
77
|
-
openLocked();
|
|
78
|
-
} catch (...) {
|
|
79
|
-
closeLocked();
|
|
80
|
-
throw;
|
|
81
|
-
}
|
|
97
|
+
reopenLocked();
|
|
82
98
|
}
|
|
83
99
|
|
|
84
100
|
SqliteDiskStore::~SqliteDiskStore() {
|
|
@@ -95,6 +111,98 @@ SqliteDiskStore& SqliteDiskStore::shared(const std::string& path) {
|
|
|
95
111
|
return *store;
|
|
96
112
|
}
|
|
97
113
|
|
|
114
|
+
bool SqliteDiskStore::isValidUtf8(const std::string& value) {
|
|
115
|
+
const auto* bytes = reinterpret_cast<const unsigned char*>(value.data());
|
|
116
|
+
const size_t size = value.size();
|
|
117
|
+
size_t index = 0;
|
|
118
|
+
while (index < size) {
|
|
119
|
+
const unsigned char lead = bytes[index];
|
|
120
|
+
size_t length = 0;
|
|
121
|
+
unsigned int minimum = 0;
|
|
122
|
+
unsigned int codePoint = 0;
|
|
123
|
+
if (lead < 0x80) {
|
|
124
|
+
index += 1;
|
|
125
|
+
continue;
|
|
126
|
+
} else if ((lead & 0xE0) == 0xC0) {
|
|
127
|
+
length = 2;
|
|
128
|
+
minimum = 0x80;
|
|
129
|
+
codePoint = lead & 0x1Fu;
|
|
130
|
+
} else if ((lead & 0xF0) == 0xE0) {
|
|
131
|
+
length = 3;
|
|
132
|
+
minimum = 0x800;
|
|
133
|
+
codePoint = lead & 0x0Fu;
|
|
134
|
+
} else if ((lead & 0xF8) == 0xF0) {
|
|
135
|
+
length = 4;
|
|
136
|
+
minimum = 0x10000;
|
|
137
|
+
codePoint = lead & 0x07u;
|
|
138
|
+
} else {
|
|
139
|
+
return false;
|
|
140
|
+
}
|
|
141
|
+
if (size - index < length) {
|
|
142
|
+
return false;
|
|
143
|
+
}
|
|
144
|
+
for (size_t offset = 1; offset < length; ++offset) {
|
|
145
|
+
const unsigned char continuation = bytes[index + offset];
|
|
146
|
+
if ((continuation & 0xC0) != 0x80) {
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
codePoint = (codePoint << 6) | (continuation & 0x3Fu);
|
|
150
|
+
}
|
|
151
|
+
if (codePoint < minimum || codePoint > 0x10FFFF || (codePoint >= 0xD800 && codePoint <= 0xDFFF)) {
|
|
152
|
+
return false;
|
|
153
|
+
}
|
|
154
|
+
index += length;
|
|
155
|
+
}
|
|
156
|
+
return true;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
void SqliteDiskStore::recreateShared(const std::string& path) {
|
|
160
|
+
std::lock_guard<std::mutex> lock(sharedMutex());
|
|
161
|
+
auto& store = sharedStore();
|
|
162
|
+
if (store && store->path() == path) {
|
|
163
|
+
store->recreate();
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
store.reset();
|
|
167
|
+
removeDatabaseFiles(path);
|
|
168
|
+
store = std::make_unique<SqliteDiskStore>(path);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
void SqliteDiskStore::removeDatabaseFiles(const std::string& path) {
|
|
172
|
+
for (const char* suffix : {"", "-wal", "-shm", "-journal"}) {
|
|
173
|
+
std::error_code error;
|
|
174
|
+
std::filesystem::remove(path + suffix, error);
|
|
175
|
+
if (error) {
|
|
176
|
+
throw std::runtime_error(
|
|
177
|
+
"NitroStorage: Disk SQLite clear failed: cannot delete the database file: " +
|
|
178
|
+
error.message()
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
void SqliteDiskStore::recreate() {
|
|
185
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
186
|
+
closeLocked();
|
|
187
|
+
removeDatabaseFiles(path_);
|
|
188
|
+
reopenLocked();
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
void SqliteDiskStore::reopenLocked() {
|
|
192
|
+
try {
|
|
193
|
+
openLocked();
|
|
194
|
+
} catch (...) {
|
|
195
|
+
closeLocked();
|
|
196
|
+
throw;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
void SqliteDiskStore::ensureOpenLocked() {
|
|
201
|
+
if (db_ == nullptr) {
|
|
202
|
+
reopenLocked();
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
98
206
|
void SqliteDiskStore::resetShared() {
|
|
99
207
|
std::lock_guard<std::mutex> lock(sharedMutex());
|
|
100
208
|
sharedStore().reset();
|
|
@@ -121,17 +229,22 @@ void SqliteDiskStore::openLocked() {
|
|
|
121
229
|
const int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX;
|
|
122
230
|
const int openRc = sqlite3_open_v2(path_.c_str(), &db, flags, nullptr);
|
|
123
231
|
if (openRc != SQLITE_OK) {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
232
|
+
try {
|
|
233
|
+
throwSqlite(db, "open", openRc);
|
|
234
|
+
} catch (...) {
|
|
235
|
+
if (db != nullptr) {
|
|
236
|
+
sqlite3_close(db);
|
|
237
|
+
}
|
|
238
|
+
throw;
|
|
127
239
|
}
|
|
128
|
-
throwSqlite(nullptr, "open", code);
|
|
129
240
|
}
|
|
130
241
|
db_ = db;
|
|
131
242
|
sqlite3_busy_timeout(db_, 5000);
|
|
132
243
|
execLocked("PRAGMA journal_mode=WAL;");
|
|
133
244
|
execLocked("PRAGMA synchronous=NORMAL;");
|
|
134
245
|
execLocked("PRAGMA temp_store=MEMORY;");
|
|
246
|
+
execLocked("PRAGMA wal_autocheckpoint=1000;");
|
|
247
|
+
execLocked("PRAGMA journal_size_limit=32768;");
|
|
135
248
|
execLocked(
|
|
136
249
|
"CREATE TABLE IF NOT EXISTS kv ("
|
|
137
250
|
"key TEXT PRIMARY KEY NOT NULL,"
|
|
@@ -149,7 +262,9 @@ void SqliteDiskStore::openLocked() {
|
|
|
149
262
|
removeStmt_ = prepareLocked("DELETE FROM kv WHERE key = ?1;");
|
|
150
263
|
hasStmt_ = prepareLocked("SELECT 1 FROM kv WHERE key = ?1 LIMIT 1;");
|
|
151
264
|
keysStmt_ = prepareLocked("SELECT key FROM kv;");
|
|
152
|
-
|
|
265
|
+
prefixRangeStmt_ = prepareLocked("SELECT key FROM kv WHERE key >= ?1 AND key < ?2;");
|
|
266
|
+
prefixFromStmt_ = prepareLocked("SELECT key FROM kv WHERE key >= ?1;");
|
|
267
|
+
prefixLikeStmt_ = prepareLocked("SELECT key FROM kv WHERE key LIKE ?1 ESCAPE '\\';");
|
|
153
268
|
sizeStmt_ = prepareLocked("SELECT COUNT(*) FROM kv;");
|
|
154
269
|
clearStmt_ = prepareLocked("DELETE FROM kv;");
|
|
155
270
|
insertAbsentStmt_ = prepareLocked(
|
|
@@ -171,7 +286,9 @@ void SqliteDiskStore::closeLocked() {
|
|
|
171
286
|
finalize(removeStmt_);
|
|
172
287
|
finalize(hasStmt_);
|
|
173
288
|
finalize(keysStmt_);
|
|
174
|
-
finalize(
|
|
289
|
+
finalize(prefixRangeStmt_);
|
|
290
|
+
finalize(prefixFromStmt_);
|
|
291
|
+
finalize(prefixLikeStmt_);
|
|
175
292
|
finalize(sizeStmt_);
|
|
176
293
|
finalize(clearStmt_);
|
|
177
294
|
finalize(insertAbsentStmt_);
|
|
@@ -192,9 +309,16 @@ void SqliteDiskStore::execLocked(const char* sql) {
|
|
|
192
309
|
|
|
193
310
|
void SqliteDiskStore::limitPageCountForTesting(int pages) {
|
|
194
311
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
312
|
+
ensureOpenLocked();
|
|
195
313
|
execLocked(("PRAGMA max_page_count=" + std::to_string(pages) + ";").c_str());
|
|
196
314
|
}
|
|
197
315
|
|
|
316
|
+
void SqliteDiskStore::limitValueLengthForTesting(int bytes) {
|
|
317
|
+
std::lock_guard<std::mutex> lock(mutex_);
|
|
318
|
+
ensureOpenLocked();
|
|
319
|
+
sqlite3_limit(db_, SQLITE_LIMIT_LENGTH, bytes);
|
|
320
|
+
}
|
|
321
|
+
|
|
198
322
|
void SqliteDiskStore::beginLocked() {
|
|
199
323
|
execLocked("BEGIN IMMEDIATE;");
|
|
200
324
|
}
|
|
@@ -264,21 +388,25 @@ void SqliteDiskStore::removeLocked(const std::string& key) {
|
|
|
264
388
|
|
|
265
389
|
void SqliteDiskStore::set(const std::string& key, const std::string& value) {
|
|
266
390
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
391
|
+
ensureOpenLocked();
|
|
267
392
|
setLocked(key, value);
|
|
268
393
|
}
|
|
269
394
|
|
|
270
395
|
std::optional<std::string> SqliteDiskStore::get(const std::string& key) {
|
|
271
396
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
397
|
+
ensureOpenLocked();
|
|
272
398
|
return getLocked(key);
|
|
273
399
|
}
|
|
274
400
|
|
|
275
401
|
void SqliteDiskStore::remove(const std::string& key) {
|
|
276
402
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
403
|
+
ensureOpenLocked();
|
|
277
404
|
removeLocked(key);
|
|
278
405
|
}
|
|
279
406
|
|
|
280
407
|
bool SqliteDiskStore::has(const std::string& key) {
|
|
281
408
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
409
|
+
ensureOpenLocked();
|
|
282
410
|
sqlite3_reset(hasStmt_);
|
|
283
411
|
sqlite3_clear_bindings(hasStmt_);
|
|
284
412
|
bindText(hasStmt_, 1, key);
|
|
@@ -298,6 +426,7 @@ void SqliteDiskStore::setBatch(
|
|
|
298
426
|
const std::vector<std::string>& values
|
|
299
427
|
) {
|
|
300
428
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
429
|
+
ensureOpenLocked();
|
|
301
430
|
beginLocked();
|
|
302
431
|
try {
|
|
303
432
|
const size_t count = std::min(keys.size(), values.size());
|
|
@@ -315,6 +444,7 @@ std::vector<std::optional<std::string>> SqliteDiskStore::getBatch(
|
|
|
315
444
|
const std::vector<std::string>& keys
|
|
316
445
|
) {
|
|
317
446
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
447
|
+
ensureOpenLocked();
|
|
318
448
|
std::vector<std::optional<std::string>> results;
|
|
319
449
|
results.reserve(keys.size());
|
|
320
450
|
for (const auto& key : keys) {
|
|
@@ -325,6 +455,7 @@ std::vector<std::optional<std::string>> SqliteDiskStore::getBatch(
|
|
|
325
455
|
|
|
326
456
|
void SqliteDiskStore::removeBatch(const std::vector<std::string>& keys) {
|
|
327
457
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
458
|
+
ensureOpenLocked();
|
|
328
459
|
beginLocked();
|
|
329
460
|
try {
|
|
330
461
|
for (const auto& key : keys) {
|
|
@@ -339,6 +470,7 @@ void SqliteDiskStore::removeBatch(const std::vector<std::string>& keys) {
|
|
|
339
470
|
|
|
340
471
|
std::vector<std::string> SqliteDiskStore::getAllKeys() {
|
|
341
472
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
473
|
+
ensureOpenLocked();
|
|
342
474
|
sqlite3_reset(keysStmt_);
|
|
343
475
|
std::vector<std::string> keys;
|
|
344
476
|
while (true) {
|
|
@@ -363,21 +495,30 @@ std::vector<std::string> SqliteDiskStore::getAllKeys() {
|
|
|
363
495
|
|
|
364
496
|
std::vector<std::string> SqliteDiskStore::getKeysByPrefix(const std::string& prefix) {
|
|
365
497
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
498
|
+
ensureOpenLocked();
|
|
499
|
+
const bool useRange = isValidUtf8(prefix);
|
|
500
|
+
const auto upperBound = useRange ? prefixUpperBound(prefix) : std::nullopt;
|
|
501
|
+
sqlite3_stmt* stmt = !useRange
|
|
502
|
+
? prefixLikeStmt_
|
|
503
|
+
: upperBound.has_value() ? prefixRangeStmt_ : prefixFromStmt_;
|
|
504
|
+
sqlite3_reset(stmt);
|
|
505
|
+
sqlite3_clear_bindings(stmt);
|
|
506
|
+
bindText(stmt, 1, useRange ? prefix : escapeLikePrefix(prefix));
|
|
507
|
+
if (upperBound.has_value()) {
|
|
508
|
+
bindText(stmt, 2, *upperBound);
|
|
509
|
+
}
|
|
369
510
|
std::vector<std::string> keys;
|
|
370
511
|
while (true) {
|
|
371
|
-
const int rc = sqlite3_step(
|
|
512
|
+
const int rc = sqlite3_step(stmt);
|
|
372
513
|
if (rc == SQLITE_DONE) {
|
|
373
514
|
break;
|
|
374
515
|
}
|
|
375
516
|
if (rc != SQLITE_ROW) {
|
|
376
|
-
sqlite3_reset(
|
|
517
|
+
sqlite3_reset(stmt);
|
|
377
518
|
throwSqlite(db_, "getKeysByPrefix", rc);
|
|
378
519
|
}
|
|
379
|
-
const unsigned char* text = sqlite3_column_text(
|
|
380
|
-
const int bytes = sqlite3_column_bytes(
|
|
520
|
+
const unsigned char* text = sqlite3_column_text(stmt, 0);
|
|
521
|
+
const int bytes = sqlite3_column_bytes(stmt, 0);
|
|
381
522
|
std::string key(
|
|
382
523
|
text != nullptr ? reinterpret_cast<const char*>(text) : "",
|
|
383
524
|
static_cast<size_t>(bytes)
|
|
@@ -386,12 +527,13 @@ std::vector<std::string> SqliteDiskStore::getKeysByPrefix(const std::string& pre
|
|
|
386
527
|
keys.push_back(std::move(key));
|
|
387
528
|
}
|
|
388
529
|
}
|
|
389
|
-
sqlite3_reset(
|
|
530
|
+
sqlite3_reset(stmt);
|
|
390
531
|
return keys;
|
|
391
532
|
}
|
|
392
533
|
|
|
393
534
|
size_t SqliteDiskStore::size() {
|
|
394
535
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
536
|
+
ensureOpenLocked();
|
|
395
537
|
sqlite3_reset(sizeStmt_);
|
|
396
538
|
const int rc = sqlite3_step(sizeStmt_);
|
|
397
539
|
if (rc != SQLITE_ROW) {
|
|
@@ -405,6 +547,7 @@ size_t SqliteDiskStore::size() {
|
|
|
405
547
|
|
|
406
548
|
void SqliteDiskStore::clear() {
|
|
407
549
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
550
|
+
ensureOpenLocked();
|
|
408
551
|
sqlite3_reset(clearStmt_);
|
|
409
552
|
const int rc = sqlite3_step(clearStmt_);
|
|
410
553
|
sqlite3_reset(clearStmt_);
|
|
@@ -463,6 +606,7 @@ void SqliteDiskStore::migrateIfAbsent(
|
|
|
463
606
|
return;
|
|
464
607
|
}
|
|
465
608
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
609
|
+
ensureOpenLocked();
|
|
466
610
|
beginLocked();
|
|
467
611
|
try {
|
|
468
612
|
insertAbsentLocked(entries);
|
|
@@ -475,6 +619,7 @@ void SqliteDiskStore::migrateIfAbsent(
|
|
|
475
619
|
|
|
476
620
|
bool SqliteDiskStore::hasMigrationMarker(const std::string& name) {
|
|
477
621
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
622
|
+
ensureOpenLocked();
|
|
478
623
|
return hasMigrationMarkerLocked(name);
|
|
479
624
|
}
|
|
480
625
|
|
|
@@ -483,6 +628,7 @@ void SqliteDiskStore::migrateOnce(
|
|
|
483
628
|
const std::vector<std::pair<std::string, std::string>>& entries
|
|
484
629
|
) {
|
|
485
630
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
631
|
+
ensureOpenLocked();
|
|
486
632
|
beginLocked();
|
|
487
633
|
try {
|
|
488
634
|
if (hasMigrationMarkerLocked(name)) {
|