sliftutils 1.6.0 → 1.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +32 -9
- package/package.json +2 -2
- package/storage/BulkDatabase2/BulkDatabase2.d.ts +13 -7
- package/storage/BulkDatabase2/BulkDatabase2.ts +13 -7
- package/storage/BulkDatabase2/BulkDatabaseBase.d.ts +13 -2
- package/storage/BulkDatabase2/BulkDatabaseBase.ts +70 -27
- package/storage/BulkDatabase2/mergeLock.d.ts +6 -0
- package/storage/BulkDatabase2/mergeLock.ts +36 -12
- package/storage/remoteFileServer.ts +7 -2
- package/yarn.lock +4 -4
package/index.d.ts
CHANGED
|
@@ -797,9 +797,9 @@ declare module "sliftutils/render-utils/observer" {
|
|
|
797
797
|
}
|
|
798
798
|
|
|
799
799
|
declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
|
|
800
|
-
import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config, BulkFileInfoListing } from "./BulkDatabaseBase";
|
|
800
|
+
import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config, BulkFileInfoListing, MergeAttemptResult } from "./BulkDatabaseBase";
|
|
801
801
|
export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
|
|
802
|
-
export type { ReactiveDeps, StorageFactory, BulkDatabase2Config, BulkFileDetails, BulkFileEntry, BulkFileInfoListing } from "./BulkDatabaseBase";
|
|
802
|
+
export type { ReactiveDeps, StorageFactory, BulkDatabase2Config, BulkFileDetails, BulkFileEntry, BulkFileInfoListing, MergeAttemptResult, MergeSkipReason } from "./BulkDatabaseBase";
|
|
803
803
|
/** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
|
|
804
804
|
export type BulkColumnInfo = {
|
|
805
805
|
column: string;
|
|
@@ -915,8 +915,13 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
|
|
|
915
915
|
* size/fragmentation and deciding whether to call tryMergeNow()/compact().
|
|
916
916
|
*/
|
|
917
917
|
getFileInfo(): Promise<BulkFileInfoListing>;
|
|
918
|
-
/**
|
|
919
|
-
|
|
918
|
+
/**
|
|
919
|
+
* Consolidate on-disk files. Optional to call; the database also does this in the background.
|
|
920
|
+
* Returns whether anything was merged, or (via skipReason) why the pass never ran — another merge
|
|
921
|
+
* in flight, another tab/process holding the merge lock (with who holds it and when the lock
|
|
922
|
+
* expires), or nothing on disk to compact.
|
|
923
|
+
*/
|
|
924
|
+
compact(): Promise<MergeAttemptResult>;
|
|
920
925
|
/**
|
|
921
926
|
* Whether this collection's storage is served over the network (a remote server) rather than local
|
|
922
927
|
* disk. Apps can branch on this to adapt to the higher latency. Note: over the network the database
|
|
@@ -934,10 +939,11 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
|
|
|
934
939
|
/**
|
|
935
940
|
* Run one merge pass now (the same policy the database runs on a timer): consolidate recent
|
|
936
941
|
* fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and
|
|
937
|
-
* whether it bailed because another tab/process holds the merge lock
|
|
938
|
-
* (e.g. every 30 minutes) and tell "nothing to do" from
|
|
942
|
+
* whether it bailed because another tab/process holds the merge lock (including the lock's holder
|
|
943
|
+
* and expiry) — so a scheduler can call this (e.g. every 30 minutes) and tell "nothing to do" from
|
|
944
|
+
* "someone else is already merging".
|
|
939
945
|
*/
|
|
940
|
-
tryMergeNow(): Promise<
|
|
946
|
+
tryMergeNow(): Promise<MergeAttemptResult>;
|
|
941
947
|
/** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
|
|
942
948
|
* most callers want compact() or tryMergeNow(). */
|
|
943
949
|
merge(timeLo: number, timeHi: number): Promise<void>;
|
|
@@ -989,6 +995,13 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
|
|
|
989
995
|
export type BulkDatabase2Config = {
|
|
990
996
|
maxTriggerThrottleMs?: number;
|
|
991
997
|
};
|
|
998
|
+
export type MergeSkipReason = "mergeInFlight" | "tabLockHeld" | "fileLockHeld" | "nothingToMerge";
|
|
999
|
+
export type MergeAttemptResult = {
|
|
1000
|
+
merged: boolean;
|
|
1001
|
+
skipReason?: MergeSkipReason;
|
|
1002
|
+
lockHolderId?: string;
|
|
1003
|
+
lockExpiresInMs?: number;
|
|
1004
|
+
};
|
|
992
1005
|
export declare class BulkDatabaseBase<T extends {
|
|
993
1006
|
key: string;
|
|
994
1007
|
}> {
|
|
@@ -1012,6 +1025,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
|
|
|
1012
1025
|
private currentStreamFileName;
|
|
1013
1026
|
private currentStreamFileBytes;
|
|
1014
1027
|
private mergeInFlight;
|
|
1028
|
+
private lastMergeSkipLogMs;
|
|
1015
1029
|
private streamRowsOnDisk;
|
|
1016
1030
|
private streamBytesOnDisk;
|
|
1017
1031
|
private fileSetPollTimer;
|
|
@@ -1062,8 +1076,11 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
|
|
|
1062
1076
|
private processMarkers;
|
|
1063
1077
|
private writeBulkFile;
|
|
1064
1078
|
private maybeMerge;
|
|
1065
|
-
|
|
1066
|
-
|
|
1079
|
+
private mergeSkip;
|
|
1080
|
+
private runLockedMerge;
|
|
1081
|
+
private tryMergeThrottled;
|
|
1082
|
+
tryMergeNow(): Promise<MergeAttemptResult>;
|
|
1083
|
+
compact(): Promise<MergeAttemptResult>;
|
|
1067
1084
|
merge(timeLo: number, timeHi: number): Promise<void>;
|
|
1068
1085
|
private readBulkHeader;
|
|
1069
1086
|
private fileLogicalSize;
|
|
@@ -1563,8 +1580,14 @@ declare module "sliftutils/storage/BulkDatabase2/blockCache" {
|
|
|
1563
1580
|
|
|
1564
1581
|
declare module "sliftutils/storage/BulkDatabase2/mergeLock" {
|
|
1565
1582
|
import type { FileStorage } from "../FileFolderAPI";
|
|
1583
|
+
export type MergeLockInfo = {
|
|
1584
|
+
holderId: string;
|
|
1585
|
+
expiresInMs: number;
|
|
1586
|
+
};
|
|
1566
1587
|
export declare function tryAcquireMergeLock(collection: string, holderId: string): boolean;
|
|
1588
|
+
export declare function peekMergeLock(collection: string): MergeLockInfo | undefined;
|
|
1567
1589
|
export declare function releaseMergeLock(collection: string, holderId: string): void;
|
|
1590
|
+
export declare function peekMergeFileLock(storage: FileStorage): Promise<MergeLockInfo | undefined>;
|
|
1568
1591
|
export declare function tryAcquireMergeFileLock(storage: FileStorage, holderId: string): Promise<boolean>;
|
|
1569
1592
|
export declare function startMergeFileLockHeartbeat(storage: FileStorage, holderId: string): () => void;
|
|
1570
1593
|
export declare function releaseMergeFileLock(storage: FileStorage, holderId: string): Promise<void>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sliftutils",
|
|
3
|
-
"version": "1.6.
|
|
3
|
+
"version": "1.6.2",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"files": [
|
|
@@ -56,7 +56,7 @@
|
|
|
56
56
|
"node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
|
|
57
57
|
"preact-old-types": "^10.28.1",
|
|
58
58
|
"shell-quote": "^1.8.3",
|
|
59
|
-
"socket-function": "^1.1.
|
|
59
|
+
"socket-function": "^1.1.48",
|
|
60
60
|
"typenode": "*",
|
|
61
61
|
"typesafecss": "*",
|
|
62
62
|
"ws": "^8.18.3",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config, BulkFileInfoListing } from "./BulkDatabaseBase";
|
|
1
|
+
import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config, BulkFileInfoListing, MergeAttemptResult } from "./BulkDatabaseBase";
|
|
2
2
|
export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
|
|
3
|
-
export type { ReactiveDeps, StorageFactory, BulkDatabase2Config, BulkFileDetails, BulkFileEntry, BulkFileInfoListing } from "./BulkDatabaseBase";
|
|
3
|
+
export type { ReactiveDeps, StorageFactory, BulkDatabase2Config, BulkFileDetails, BulkFileEntry, BulkFileInfoListing, MergeAttemptResult, MergeSkipReason } from "./BulkDatabaseBase";
|
|
4
4
|
/** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
|
|
5
5
|
export type BulkColumnInfo = {
|
|
6
6
|
column: string;
|
|
@@ -116,8 +116,13 @@ export interface IBulkDatabase2<T extends {
|
|
|
116
116
|
* size/fragmentation and deciding whether to call tryMergeNow()/compact().
|
|
117
117
|
*/
|
|
118
118
|
getFileInfo(): Promise<BulkFileInfoListing>;
|
|
119
|
-
/**
|
|
120
|
-
|
|
119
|
+
/**
|
|
120
|
+
* Consolidate on-disk files. Optional to call; the database also does this in the background.
|
|
121
|
+
* Returns whether anything was merged, or (via skipReason) why the pass never ran — another merge
|
|
122
|
+
* in flight, another tab/process holding the merge lock (with who holds it and when the lock
|
|
123
|
+
* expires), or nothing on disk to compact.
|
|
124
|
+
*/
|
|
125
|
+
compact(): Promise<MergeAttemptResult>;
|
|
121
126
|
/**
|
|
122
127
|
* Whether this collection's storage is served over the network (a remote server) rather than local
|
|
123
128
|
* disk. Apps can branch on this to adapt to the higher latency. Note: over the network the database
|
|
@@ -135,10 +140,11 @@ export interface IBulkDatabase2<T extends {
|
|
|
135
140
|
/**
|
|
136
141
|
* Run one merge pass now (the same policy the database runs on a timer): consolidate recent
|
|
137
142
|
* fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and
|
|
138
|
-
* whether it bailed because another tab/process holds the merge lock
|
|
139
|
-
* (e.g. every 30 minutes) and tell "nothing to do" from
|
|
143
|
+
* whether it bailed because another tab/process holds the merge lock (including the lock's holder
|
|
144
|
+
* and expiry) — so a scheduler can call this (e.g. every 30 minutes) and tell "nothing to do" from
|
|
145
|
+
* "someone else is already merging".
|
|
140
146
|
*/
|
|
141
|
-
tryMergeNow(): Promise<
|
|
147
|
+
tryMergeNow(): Promise<MergeAttemptResult>;
|
|
142
148
|
/** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
|
|
143
149
|
* most callers want compact() or tryMergeNow(). */
|
|
144
150
|
merge(timeLo: number, timeHi: number): Promise<void>;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { getFileStorageNested2 } from "../FileFolderAPI";
|
|
2
2
|
import { observable, runInAction, onBecomeObserved, onBecomeUnobserved } from "../../render-utils/mobxTyped";
|
|
3
|
-
import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config, BulkFileInfoListing } from "./BulkDatabaseBase";
|
|
3
|
+
import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config, BulkFileInfoListing, MergeAttemptResult } from "./BulkDatabaseBase";
|
|
4
4
|
|
|
5
5
|
export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
|
|
6
|
-
export type { ReactiveDeps, StorageFactory, BulkDatabase2Config, BulkFileDetails, BulkFileEntry, BulkFileInfoListing } from "./BulkDatabaseBase";
|
|
6
|
+
export type { ReactiveDeps, StorageFactory, BulkDatabase2Config, BulkFileDetails, BulkFileEntry, BulkFileInfoListing, MergeAttemptResult, MergeSkipReason } from "./BulkDatabaseBase";
|
|
7
7
|
|
|
8
8
|
/** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
|
|
9
9
|
export type BulkColumnInfo = { column: string; byteSize: number };
|
|
@@ -106,8 +106,13 @@ export interface IBulkDatabase2<T extends { key: string }> {
|
|
|
106
106
|
*/
|
|
107
107
|
getFileInfo(): Promise<BulkFileInfoListing>;
|
|
108
108
|
|
|
109
|
-
/**
|
|
110
|
-
|
|
109
|
+
/**
|
|
110
|
+
* Consolidate on-disk files. Optional to call; the database also does this in the background.
|
|
111
|
+
* Returns whether anything was merged, or (via skipReason) why the pass never ran — another merge
|
|
112
|
+
* in flight, another tab/process holding the merge lock (with who holds it and when the lock
|
|
113
|
+
* expires), or nothing on disk to compact.
|
|
114
|
+
*/
|
|
115
|
+
compact(): Promise<MergeAttemptResult>;
|
|
111
116
|
|
|
112
117
|
/**
|
|
113
118
|
* Whether this collection's storage is served over the network (a remote server) rather than local
|
|
@@ -128,10 +133,11 @@ export interface IBulkDatabase2<T extends { key: string }> {
|
|
|
128
133
|
/**
|
|
129
134
|
* Run one merge pass now (the same policy the database runs on a timer): consolidate recent
|
|
130
135
|
* fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and
|
|
131
|
-
* whether it bailed because another tab/process holds the merge lock
|
|
132
|
-
* (e.g. every 30 minutes) and tell "nothing to do" from
|
|
136
|
+
* whether it bailed because another tab/process holds the merge lock (including the lock's holder
|
|
137
|
+
* and expiry) — so a scheduler can call this (e.g. every 30 minutes) and tell "nothing to do" from
|
|
138
|
+
* "someone else is already merging".
|
|
133
139
|
*/
|
|
134
|
-
tryMergeNow(): Promise<
|
|
140
|
+
tryMergeNow(): Promise<MergeAttemptResult>;
|
|
135
141
|
|
|
136
142
|
/** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
|
|
137
143
|
* most callers want compact() or tryMergeNow(). */
|
|
@@ -27,6 +27,13 @@ export type StorageFactory = (path: string) => Promise<FileStorage>;
|
|
|
27
27
|
export type BulkDatabase2Config = {
|
|
28
28
|
maxTriggerThrottleMs?: number;
|
|
29
29
|
};
|
|
30
|
+
export type MergeSkipReason = "mergeInFlight" | "tabLockHeld" | "fileLockHeld" | "nothingToMerge";
|
|
31
|
+
export type MergeAttemptResult = {
|
|
32
|
+
merged: boolean;
|
|
33
|
+
skipReason?: MergeSkipReason;
|
|
34
|
+
lockHolderId?: string;
|
|
35
|
+
lockExpiresInMs?: number;
|
|
36
|
+
};
|
|
30
37
|
export declare class BulkDatabaseBase<T extends {
|
|
31
38
|
key: string;
|
|
32
39
|
}> {
|
|
@@ -50,6 +57,7 @@ export declare class BulkDatabaseBase<T extends {
|
|
|
50
57
|
private currentStreamFileName;
|
|
51
58
|
private currentStreamFileBytes;
|
|
52
59
|
private mergeInFlight;
|
|
60
|
+
private lastMergeSkipLogMs;
|
|
53
61
|
private streamRowsOnDisk;
|
|
54
62
|
private streamBytesOnDisk;
|
|
55
63
|
private fileSetPollTimer;
|
|
@@ -100,8 +108,11 @@ export declare class BulkDatabaseBase<T extends {
|
|
|
100
108
|
private processMarkers;
|
|
101
109
|
private writeBulkFile;
|
|
102
110
|
private maybeMerge;
|
|
103
|
-
|
|
104
|
-
|
|
111
|
+
private mergeSkip;
|
|
112
|
+
private runLockedMerge;
|
|
113
|
+
private tryMergeThrottled;
|
|
114
|
+
tryMergeNow(): Promise<MergeAttemptResult>;
|
|
115
|
+
compact(): Promise<MergeAttemptResult>;
|
|
105
116
|
merge(timeLo: number, timeHi: number): Promise<void>;
|
|
106
117
|
private readBulkHeader;
|
|
107
118
|
private fileLogicalSize;
|
|
@@ -18,7 +18,7 @@ import { blue, magenta } from "socket-function/src/formatting/logColors";
|
|
|
18
18
|
import { STREAM_EXTENSION, frameDeletes, frameRows, streamReaderFromEntries } from "./streamLog";
|
|
19
19
|
import { broadcast as syncBroadcast, broadcastSeal as syncBroadcastSeal, connect as syncConnect, isSyncSupported, RemoteWrite } from "./syncClient";
|
|
20
20
|
import { DELETED } from "./WriteOverlay";
|
|
21
|
-
import { releaseMergeFileLock, releaseMergeLock, startMergeFileLockHeartbeat, tryAcquireMergeFileLock, tryAcquireMergeLock } from "./mergeLock";
|
|
21
|
+
import { MergeLockInfo, peekMergeFileLock, peekMergeLock, releaseMergeFileLock, releaseMergeLock, startMergeFileLockHeartbeat, tryAcquireMergeFileLock, tryAcquireMergeLock } from "./mergeLock";
|
|
22
22
|
import { markerExclusions, processDeleteMarkers, readDeleteMarkers, writeDeleteMarker } from "./mergeMarkers";
|
|
23
23
|
import {
|
|
24
24
|
BulkFileInfo,
|
|
@@ -49,6 +49,9 @@ const DUP_THRESHOLD = 0.4;
|
|
|
49
49
|
const DEDUP_TRIGGER_BYTES = 512 * 1024 * 1024;
|
|
50
50
|
const DEDUP_TRIGGER_FRACTION = 0.5;
|
|
51
51
|
const WRITE_FLUSH_FIRST_STEP_MS = 250;
|
|
52
|
+
// Skip logs are throttled per collection: the background write path retries merges about once a second,
|
|
53
|
+
// so an unthrottled log would repeat for the whole duration of another tab's merge.
|
|
54
|
+
const MERGE_SKIP_LOG_INTERVAL_MS = 30 * 1000;
|
|
52
55
|
|
|
53
56
|
export const bulkDatabase2Timing = {
|
|
54
57
|
streamSealAgeMs: 10 * 60 * 60 * 1000,
|
|
@@ -67,7 +70,7 @@ export const bulkDatabase2Timing = {
|
|
|
67
70
|
// the whole stream file per write.
|
|
68
71
|
writeFlushMaxDelayMs: isNode() ? 0 : 15 * 1000,
|
|
69
72
|
fileSetPollIntervalMs: 30 * 60 * 1000,
|
|
70
|
-
memoryFlushHeapBytes:
|
|
73
|
+
memoryFlushHeapBytes: 5 * 1024 * 1024 * 1024,
|
|
71
74
|
memoryFlushMinCollectionBytes: 100 * 1024 * 1024,
|
|
72
75
|
memoryFlushThrottleMs: 15 * 60 * 1000,
|
|
73
76
|
};
|
|
@@ -104,6 +107,26 @@ export type BulkDatabase2Config = {
|
|
|
104
107
|
maxTriggerThrottleMs?: number;
|
|
105
108
|
};
|
|
106
109
|
|
|
110
|
+
export type MergeSkipReason =
|
|
111
|
+
// This instance is already mid-merge.
|
|
112
|
+
| "mergeInFlight"
|
|
113
|
+
// Another same-origin tab holds the localStorage merge lock.
|
|
114
|
+
| "tabLockHeld"
|
|
115
|
+
// Another process holds the cross-process .merge-lock file (or won the settle race for it).
|
|
116
|
+
| "fileLockHeld"
|
|
117
|
+
// compact() only: the collection has no files on disk.
|
|
118
|
+
| "nothingToMerge";
|
|
119
|
+
|
|
120
|
+
// What a compact()/tryMergeNow() call did. skipReason is set when the pass never ran; for the lock
|
|
121
|
+
// reasons, lockHolderId/lockExpiresInMs report who holds the lock and how long until it goes stale
|
|
122
|
+
// (so a scheduler knows when retrying could succeed).
|
|
123
|
+
export type MergeAttemptResult = {
|
|
124
|
+
merged: boolean;
|
|
125
|
+
skipReason?: MergeSkipReason;
|
|
126
|
+
lockHolderId?: string;
|
|
127
|
+
lockExpiresInMs?: number;
|
|
128
|
+
};
|
|
129
|
+
|
|
107
130
|
let networkCompactionEnabled = false;
|
|
108
131
|
|
|
109
132
|
let fileNameCounter = 0;
|
|
@@ -237,6 +260,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
237
260
|
// tryMergeNow calls) return immediately instead of queueing. Keeps the visibility-timer firings
|
|
238
261
|
// from stacking behind a long merge.
|
|
239
262
|
private mergeInFlight = false;
|
|
263
|
+
private lastMergeSkipLogMs = 0;
|
|
240
264
|
// Running counters of stream-tier rows + bytes on disk. Seeded from each LoadedIndex build, then
|
|
241
265
|
// incremented per flush so the fold-trigger checks current data without an extra directory listing.
|
|
242
266
|
private streamRowsOnDisk = 0;
|
|
@@ -614,58 +638,77 @@ export class BulkDatabaseBase<T extends { key: string }> {
|
|
|
614
638
|
if (!await this.automaticCompactionAllowed()) return;
|
|
615
639
|
if (opts.onlyIfStreamHeavy && !this.streamNeedsFold()) return;
|
|
616
640
|
try {
|
|
617
|
-
await this.
|
|
641
|
+
await this.tryMergeThrottled();
|
|
618
642
|
} catch (e) {
|
|
619
643
|
console.warn(`${this.name}: background merge failed: ${(e as Error).message}`);
|
|
620
644
|
}
|
|
621
645
|
}
|
|
622
646
|
|
|
623
|
-
//
|
|
624
|
-
//
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
if (
|
|
628
|
-
|
|
647
|
+
// Build the "we didn't merge" result and log it (throttled — see MERGE_SKIP_LOG_INTERVAL_MS). For
|
|
648
|
+
// lock reasons the log and result include the holder and time until the lock goes stale.
|
|
649
|
+
private async mergeSkip(reason: MergeSkipReason): Promise<MergeAttemptResult> {
|
|
650
|
+
let lock: MergeLockInfo | undefined;
|
|
651
|
+
if (reason === "tabLockHeld") {
|
|
652
|
+
lock = peekMergeLock(this.name);
|
|
653
|
+
} else if (reason === "fileLockHeld") {
|
|
654
|
+
lock = await peekMergeFileLock(await this.storage());
|
|
655
|
+
}
|
|
656
|
+
if (Date.now() - this.lastMergeSkipLogMs >= MERGE_SKIP_LOG_INTERVAL_MS) {
|
|
657
|
+
this.lastMergeSkipLogMs = Date.now();
|
|
658
|
+
const detail = lock && ` (held by ${lock.holderId}, expires in ${formatTime(Math.max(0, lock.expiresInMs))})` || "";
|
|
659
|
+
console.log(`${blue(this.name)} ${magenta("merge")} skipped: ${reason}${detail}`);
|
|
660
|
+
}
|
|
661
|
+
return { merged: false, skipReason: reason, lockHolderId: lock?.holderId, lockExpiresInMs: lock?.expiresInMs };
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
// Acquire both merge locks (same-tab mergeInFlight, cross-tab localStorage, cross-process file),
|
|
665
|
+
// run the pass, release. Skips (with the reason) instead of waiting when any of them is held.
|
|
666
|
+
private async runLockedMerge(run: () => Promise<MergeAttemptResult>): Promise<MergeAttemptResult> {
|
|
667
|
+
if (this.mergeInFlight) return await this.mergeSkip("mergeInFlight");
|
|
668
|
+
if (!tryAcquireMergeLock(this.name, writerId)) return await this.mergeSkip("tabLockHeld");
|
|
629
669
|
const storage = await this.storage();
|
|
630
670
|
const haveFileLock = await tryAcquireMergeFileLock(storage, writerId);
|
|
631
671
|
if (!haveFileLock) {
|
|
632
672
|
releaseMergeLock(this.name, writerId);
|
|
633
|
-
return;
|
|
673
|
+
return await this.mergeSkip("fileLockHeld");
|
|
634
674
|
}
|
|
635
675
|
this.mergeInFlight = true;
|
|
636
676
|
const stopHeartbeat = startMergeFileLockHeartbeat(storage, writerId);
|
|
637
677
|
try {
|
|
638
|
-
await
|
|
678
|
+
return await run();
|
|
639
679
|
} finally {
|
|
640
680
|
stopHeartbeat();
|
|
641
681
|
await releaseMergeFileLock(storage, writerId);
|
|
642
682
|
releaseMergeLock(this.name, writerId);
|
|
643
683
|
this.mergeInFlight = false;
|
|
644
684
|
}
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
// The background path (maybeMerge) goes through this so write bursts coalesce; explicit callers use
|
|
688
|
+
// tryMergeNow directly and get the per-call result.
|
|
689
|
+
private tryMergeThrottled = throttleFunction(1000, async () => {
|
|
690
|
+
await this.tryMergeNow();
|
|
645
691
|
});
|
|
646
692
|
|
|
647
|
-
public async
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
693
|
+
public async tryMergeNow(): Promise<MergeAttemptResult> {
|
|
694
|
+
return await this.runLockedMerge(async () => {
|
|
695
|
+
const merged = await this.testMergeINTERNAL_DO_NOT_CALL();
|
|
696
|
+
return { merged };
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
public async compact(): Promise<MergeAttemptResult> {
|
|
701
|
+
return await this.runLockedMerge(async () => {
|
|
656
702
|
await this.flushPending();
|
|
657
703
|
syncBroadcastSeal(this.name);
|
|
658
704
|
this.streamFileName = undefined;
|
|
659
705
|
const { bulkFiles, streamFiles } = await this.listFiles();
|
|
706
|
+
if (bulkFiles.length + streamFiles.length < 1) return await this.mergeSkip("nothingToMerge");
|
|
660
707
|
// compact() folds every file → no older data survives outside it → surviving tombstones
|
|
661
708
|
// can be dropped (nothing left to suppress).
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
await releaseMergeFileLock(storage, writerId);
|
|
666
|
-
releaseMergeLock(this.name, writerId);
|
|
667
|
-
this.mergeInFlight = false;
|
|
668
|
-
}
|
|
709
|
+
const merged = await this.mergeFileSet(bulkFiles, streamFiles, true);
|
|
710
|
+
return { merged };
|
|
711
|
+
});
|
|
669
712
|
}
|
|
670
713
|
|
|
671
714
|
public async merge(timeLo: number, timeHi: number): Promise<void> {
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import type { FileStorage } from "../FileFolderAPI";
|
|
2
|
+
export type MergeLockInfo = {
|
|
3
|
+
holderId: string;
|
|
4
|
+
expiresInMs: number;
|
|
5
|
+
};
|
|
2
6
|
export declare function tryAcquireMergeLock(collection: string, holderId: string): boolean;
|
|
7
|
+
export declare function peekMergeLock(collection: string): MergeLockInfo | undefined;
|
|
3
8
|
export declare function releaseMergeLock(collection: string, holderId: string): void;
|
|
9
|
+
export declare function peekMergeFileLock(storage: FileStorage): Promise<MergeLockInfo | undefined>;
|
|
4
10
|
export declare function tryAcquireMergeFileLock(storage: FileStorage, holderId: string): Promise<boolean>;
|
|
5
11
|
export declare function startMergeFileLockHeartbeat(storage: FileStorage, holderId: string): () => void;
|
|
6
12
|
export declare function releaseMergeFileLock(storage: FileStorage, holderId: string): Promise<void>;
|
|
@@ -17,6 +17,16 @@ import type { FileStorage } from "../FileFolderAPI";
|
|
|
17
17
|
|
|
18
18
|
const LOCK_TTL_MS = 30 * 1000;
|
|
19
19
|
|
|
20
|
+
export type MergeLockInfo = { holderId: string; expiresInMs: number };
|
|
21
|
+
|
|
22
|
+
function parseLockToken(token: string): { holderId: string; time: number } | undefined {
|
|
23
|
+
const colon = token.lastIndexOf(":");
|
|
24
|
+
if (colon < 0) return undefined;
|
|
25
|
+
const time = parseInt(token.slice(colon + 1), 10);
|
|
26
|
+
if (!Number.isFinite(time)) return undefined;
|
|
27
|
+
return { holderId: token.slice(0, colon), time };
|
|
28
|
+
}
|
|
29
|
+
|
|
20
30
|
function getLocalStorage(): Storage | undefined {
|
|
21
31
|
try {
|
|
22
32
|
return typeof localStorage !== "undefined" ? localStorage : undefined;
|
|
@@ -40,8 +50,8 @@ export function tryAcquireMergeLock(collection: string, holderId: string): boole
|
|
|
40
50
|
const now = Date.now();
|
|
41
51
|
const existing = ls.getItem(key);
|
|
42
52
|
if (existing && !existing.startsWith(holderId + ":")) {
|
|
43
|
-
const
|
|
44
|
-
if (
|
|
53
|
+
const parsed = parseLockToken(existing);
|
|
54
|
+
if (parsed && now - parsed.time < LOCK_TTL_MS) return false;
|
|
45
55
|
}
|
|
46
56
|
const token = holderId + ":" + now;
|
|
47
57
|
ls.setItem(key, token);
|
|
@@ -49,6 +59,18 @@ export function tryAcquireMergeLock(collection: string, holderId: string): boole
|
|
|
49
59
|
return ls.getItem(key) === token;
|
|
50
60
|
}
|
|
51
61
|
|
|
62
|
+
// Non-mutating look at the tab lock: who holds it and how long until it goes stale (negative = already
|
|
63
|
+
// stale). For logging why a merge was skipped.
|
|
64
|
+
export function peekMergeLock(collection: string): MergeLockInfo | undefined {
|
|
65
|
+
const ls = getLocalStorage();
|
|
66
|
+
if (!ls) return undefined;
|
|
67
|
+
const existing = ls.getItem(lockKey(collection));
|
|
68
|
+
if (!existing) return undefined;
|
|
69
|
+
const parsed = parseLockToken(existing);
|
|
70
|
+
if (!parsed) return undefined;
|
|
71
|
+
return { holderId: parsed.holderId, expiresInMs: parsed.time + LOCK_TTL_MS - Date.now() };
|
|
72
|
+
}
|
|
73
|
+
|
|
52
74
|
export function releaseMergeLock(collection: string, holderId: string): void {
|
|
53
75
|
const ls = getLocalStorage();
|
|
54
76
|
if (!ls) return;
|
|
@@ -63,19 +85,21 @@ const FILE_LOCK_TTL_MS = 5 * 60 * 1000;
|
|
|
63
85
|
const FILE_LOCK_SETTLE_MS = 15 * 1000;
|
|
64
86
|
const FILE_LOCK_HEARTBEAT_MS = 2 * 60 * 1000;
|
|
65
87
|
|
|
66
|
-
async function readMergeFileLock(storage: FileStorage): Promise<{
|
|
88
|
+
async function readMergeFileLock(storage: FileStorage): Promise<{ holderId: string; time: number } | undefined> {
|
|
67
89
|
try {
|
|
68
90
|
const buf = await storage.get(FILE_LOCK_KEY);
|
|
69
91
|
if (!buf) return undefined;
|
|
70
|
-
|
|
71
|
-
const colon = s.lastIndexOf(":");
|
|
72
|
-
if (colon < 0) return undefined;
|
|
73
|
-
const time = parseInt(s.slice(colon + 1), 10);
|
|
74
|
-
if (!Number.isFinite(time)) return undefined;
|
|
75
|
-
return { id: s.slice(0, colon), time };
|
|
92
|
+
return parseLockToken(buf.toString("utf8"));
|
|
76
93
|
} catch { return undefined; }
|
|
77
94
|
}
|
|
78
95
|
|
|
96
|
+
// Non-mutating look at the cross-process lock file (see peekMergeLock).
|
|
97
|
+
export async function peekMergeFileLock(storage: FileStorage): Promise<MergeLockInfo | undefined> {
|
|
98
|
+
const existing = await readMergeFileLock(storage);
|
|
99
|
+
if (!existing) return undefined;
|
|
100
|
+
return { holderId: existing.holderId, expiresInMs: existing.time + FILE_LOCK_TTL_MS - Date.now() };
|
|
101
|
+
}
|
|
102
|
+
|
|
79
103
|
async function writeMergeFileLock(storage: FileStorage, holderId: string, time: number): Promise<void> {
|
|
80
104
|
await storage.set(FILE_LOCK_KEY, Buffer.from(`${holderId}:${time}`, "utf8") as Buffer);
|
|
81
105
|
}
|
|
@@ -86,11 +110,11 @@ async function writeMergeFileLock(storage: FileStorage, holderId: string, time:
|
|
|
86
110
|
export async function tryAcquireMergeFileLock(storage: FileStorage, holderId: string): Promise<boolean> {
|
|
87
111
|
const now = Date.now();
|
|
88
112
|
const existing = await readMergeFileLock(storage);
|
|
89
|
-
if (existing && existing.
|
|
113
|
+
if (existing && existing.holderId !== holderId && now - existing.time < FILE_LOCK_TTL_MS) return false;
|
|
90
114
|
await writeMergeFileLock(storage, holderId, now);
|
|
91
115
|
await new Promise(r => setTimeout(r, FILE_LOCK_SETTLE_MS));
|
|
92
116
|
const after = await readMergeFileLock(storage);
|
|
93
|
-
return !!after && after.
|
|
117
|
+
return !!after && after.holderId === holderId;
|
|
94
118
|
}
|
|
95
119
|
|
|
96
120
|
// Keep the lock fresh. The merge might run longer than FILE_LOCK_TTL_MS; without periodic re-stamping,
|
|
@@ -107,6 +131,6 @@ export function startMergeFileLockHeartbeat(storage: FileStorage, holderId: stri
|
|
|
107
131
|
export async function releaseMergeFileLock(storage: FileStorage, holderId: string): Promise<void> {
|
|
108
132
|
try {
|
|
109
133
|
const existing = await readMergeFileLock(storage);
|
|
110
|
-
if (existing && existing.
|
|
134
|
+
if (existing && existing.holderId === holderId) await storage.remove(FILE_LOCK_KEY);
|
|
111
135
|
} catch { /* ignore */ }
|
|
112
136
|
}
|
|
@@ -627,8 +627,13 @@ export async function autocompactBulkDatabases(root: string): Promise<void> {
|
|
|
627
627
|
const start = Date.now();
|
|
628
628
|
console.log(` [autocompact] ${at} ${name}: starting merge`);
|
|
629
629
|
try {
|
|
630
|
-
await getCompactor(baseDir, name).tryMergeNow();
|
|
631
|
-
|
|
630
|
+
const result = await getCompactor(baseDir, name).tryMergeNow();
|
|
631
|
+
let outcome = result.merged && "merged" || "nothing to merge";
|
|
632
|
+
if (result.skipReason) {
|
|
633
|
+
outcome = `skipped: ${result.skipReason}`;
|
|
634
|
+
if (result.lockExpiresInMs !== undefined) outcome += ` (lock held by ${result.lockHolderId}, expires in ${Math.max(0, Math.round(result.lockExpiresInMs / 1000))}s)`;
|
|
635
|
+
}
|
|
636
|
+
console.log(` [autocompact] ${at} ${name}: done (${outcome}, ${Date.now() - start}ms)`);
|
|
632
637
|
} catch (e) {
|
|
633
638
|
console.warn(` [autocompact] ${at} ${name}: failed after ${Date.now() - start}ms - ${(e as Error).message}`);
|
|
634
639
|
}
|
package/yarn.lock
CHANGED
|
@@ -1944,10 +1944,10 @@ slash@^3.0.0:
|
|
|
1944
1944
|
resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
|
|
1945
1945
|
integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
|
|
1946
1946
|
|
|
1947
|
-
socket-function@^1.1.
|
|
1948
|
-
version "1.1.
|
|
1949
|
-
resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.1.
|
|
1950
|
-
integrity sha512-
|
|
1947
|
+
socket-function@^1.1.48:
|
|
1948
|
+
version "1.1.48"
|
|
1949
|
+
resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.1.48.tgz#26266a635847497e726ca49061bc604b4ba0abb1"
|
|
1950
|
+
integrity sha512-kKniLUzh3ZneNcNLFxQiiR1TCSAGeyJgHs5S0643s9SmaVZCyEdYTKYgV+u7jzDSjz81wTQVdv/DChVOQNNH3w==
|
|
1951
1951
|
dependencies:
|
|
1952
1952
|
"@types/pako" "^2.0.3"
|
|
1953
1953
|
"@types/ws" "^8.5.3"
|