sliftutils 1.6.0 → 1.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.d.ts CHANGED
@@ -797,9 +797,9 @@ declare module "sliftutils/render-utils/observer" {
797
797
  }
798
798
 
799
799
  declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
800
- import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config, BulkFileInfoListing } from "./BulkDatabaseBase";
800
+ import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config, BulkFileInfoListing, MergeAttemptResult } from "./BulkDatabaseBase";
801
801
  export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
802
- export type { ReactiveDeps, StorageFactory, BulkDatabase2Config, BulkFileDetails, BulkFileEntry, BulkFileInfoListing } from "./BulkDatabaseBase";
802
+ export type { ReactiveDeps, StorageFactory, BulkDatabase2Config, BulkFileDetails, BulkFileEntry, BulkFileInfoListing, MergeAttemptResult, MergeSkipReason } from "./BulkDatabaseBase";
803
803
  /** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
804
804
  export type BulkColumnInfo = {
805
805
  column: string;
@@ -915,8 +915,13 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
915
915
  * size/fragmentation and deciding whether to call tryMergeNow()/compact().
916
916
  */
917
917
  getFileInfo(): Promise<BulkFileInfoListing>;
918
- /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
919
- compact(): Promise<void>;
918
+ /**
919
+ * Consolidate on-disk files. Optional to call; the database also does this in the background.
920
+ * Returns whether anything was merged, or (via skipReason) why the pass never ran — another merge
921
+ * in flight, another tab/process holding the merge lock (with who holds it and when the lock
922
+ * expires), or nothing on disk to compact.
923
+ */
924
+ compact(): Promise<MergeAttemptResult>;
920
925
  /**
921
926
  * Whether this collection's storage is served over the network (a remote server) rather than local
922
927
  * disk. Apps can branch on this to adapt to the higher latency. Note: over the network the database
@@ -934,10 +939,11 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabase2" {
934
939
  /**
935
940
  * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
936
941
  * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and
937
- * whether it bailed because another tab/process holds the merge lock so a scheduler can call this
938
- * (e.g. every 30 minutes) and tell "nothing to do" from "someone else is already merging".
942
+ * whether it bailed because another tab/process holds the merge lock (including the lock's holder
943
+ * and expiry) — so a scheduler can call this (e.g. every 30 minutes) and tell "nothing to do" from
944
+ * "someone else is already merging".
939
945
  */
940
- tryMergeNow(): Promise<void>;
946
+ tryMergeNow(): Promise<MergeAttemptResult>;
941
947
  /** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
942
948
  * most callers want compact() or tryMergeNow(). */
943
949
  merge(timeLo: number, timeHi: number): Promise<void>;
@@ -989,6 +995,13 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
989
995
  export type BulkDatabase2Config = {
990
996
  maxTriggerThrottleMs?: number;
991
997
  };
998
+ export type MergeSkipReason = "mergeInFlight" | "tabLockHeld" | "fileLockHeld" | "nothingToMerge";
999
+ export type MergeAttemptResult = {
1000
+ merged: boolean;
1001
+ skipReason?: MergeSkipReason;
1002
+ lockHolderId?: string;
1003
+ lockExpiresInMs?: number;
1004
+ };
992
1005
  export declare class BulkDatabaseBase<T extends {
993
1006
  key: string;
994
1007
  }> {
@@ -1012,6 +1025,7 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
1012
1025
  private currentStreamFileName;
1013
1026
  private currentStreamFileBytes;
1014
1027
  private mergeInFlight;
1028
+ private lastMergeSkipLogMs;
1015
1029
  private streamRowsOnDisk;
1016
1030
  private streamBytesOnDisk;
1017
1031
  private fileSetPollTimer;
@@ -1062,8 +1076,11 @@ declare module "sliftutils/storage/BulkDatabase2/BulkDatabaseBase" {
1062
1076
  private processMarkers;
1063
1077
  private writeBulkFile;
1064
1078
  private maybeMerge;
1065
- tryMergeNow: () => Promise<void>;
1066
- compact(): Promise<void>;
1079
+ private mergeSkip;
1080
+ private runLockedMerge;
1081
+ private tryMergeThrottled;
1082
+ tryMergeNow(): Promise<MergeAttemptResult>;
1083
+ compact(): Promise<MergeAttemptResult>;
1067
1084
  merge(timeLo: number, timeHi: number): Promise<void>;
1068
1085
  private readBulkHeader;
1069
1086
  private fileLogicalSize;
@@ -1563,8 +1580,14 @@ declare module "sliftutils/storage/BulkDatabase2/blockCache" {
1563
1580
 
1564
1581
  declare module "sliftutils/storage/BulkDatabase2/mergeLock" {
1565
1582
  import type { FileStorage } from "../FileFolderAPI";
1583
+ export type MergeLockInfo = {
1584
+ holderId: string;
1585
+ expiresInMs: number;
1586
+ };
1566
1587
  export declare function tryAcquireMergeLock(collection: string, holderId: string): boolean;
1588
+ export declare function peekMergeLock(collection: string): MergeLockInfo | undefined;
1567
1589
  export declare function releaseMergeLock(collection: string, holderId: string): void;
1590
+ export declare function peekMergeFileLock(storage: FileStorage): Promise<MergeLockInfo | undefined>;
1568
1591
  export declare function tryAcquireMergeFileLock(storage: FileStorage, holderId: string): Promise<boolean>;
1569
1592
  export declare function startMergeFileLockHeartbeat(storage: FileStorage, holderId: string): () => void;
1570
1593
  export declare function releaseMergeFileLock(storage: FileStorage, holderId: string): Promise<void>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "sliftutils",
3
- "version": "1.6.0",
3
+ "version": "1.6.2",
4
4
  "main": "index.js",
5
5
  "license": "MIT",
6
6
  "files": [
@@ -56,7 +56,7 @@
56
56
  "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
57
57
  "preact-old-types": "^10.28.1",
58
58
  "shell-quote": "^1.8.3",
59
- "socket-function": "^1.1.42",
59
+ "socket-function": "^1.1.48",
60
60
  "typenode": "*",
61
61
  "typesafecss": "*",
62
62
  "ws": "^8.18.3",
@@ -1,6 +1,6 @@
1
- import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config, BulkFileInfoListing } from "./BulkDatabaseBase";
1
+ import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config, BulkFileInfoListing, MergeAttemptResult } from "./BulkDatabaseBase";
2
2
  export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
3
- export type { ReactiveDeps, StorageFactory, BulkDatabase2Config, BulkFileDetails, BulkFileEntry, BulkFileInfoListing } from "./BulkDatabaseBase";
3
+ export type { ReactiveDeps, StorageFactory, BulkDatabase2Config, BulkFileDetails, BulkFileEntry, BulkFileInfoListing, MergeAttemptResult, MergeSkipReason } from "./BulkDatabaseBase";
4
4
  /** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
5
5
  export type BulkColumnInfo = {
6
6
  column: string;
@@ -116,8 +116,13 @@ export interface IBulkDatabase2<T extends {
116
116
  * size/fragmentation and deciding whether to call tryMergeNow()/compact().
117
117
  */
118
118
  getFileInfo(): Promise<BulkFileInfoListing>;
119
- /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
120
- compact(): Promise<void>;
119
+ /**
120
+ * Consolidate on-disk files. Optional to call; the database also does this in the background.
121
+ * Returns whether anything was merged, or (via skipReason) why the pass never ran — another merge
122
+ * in flight, another tab/process holding the merge lock (with who holds it and when the lock
123
+ * expires), or nothing on disk to compact.
124
+ */
125
+ compact(): Promise<MergeAttemptResult>;
121
126
  /**
122
127
  * Whether this collection's storage is served over the network (a remote server) rather than local
123
128
  * disk. Apps can branch on this to adapt to the higher latency. Note: over the network the database
@@ -135,10 +140,11 @@ export interface IBulkDatabase2<T extends {
135
140
  /**
136
141
  * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
137
142
  * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and
138
- * whether it bailed because another tab/process holds the merge lock so a scheduler can call this
139
- * (e.g. every 30 minutes) and tell "nothing to do" from "someone else is already merging".
143
+ * whether it bailed because another tab/process holds the merge lock (including the lock's holder
144
+ * and expiry) — so a scheduler can call this (e.g. every 30 minutes) and tell "nothing to do" from
145
+ * "someone else is already merging".
140
146
  */
141
- tryMergeNow(): Promise<void>;
147
+ tryMergeNow(): Promise<MergeAttemptResult>;
142
148
  /** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
143
149
  * most callers want compact() or tryMergeNow(). */
144
150
  merge(timeLo: number, timeHi: number): Promise<void>;
@@ -1,9 +1,9 @@
1
1
  import { getFileStorageNested2 } from "../FileFolderAPI";
2
2
  import { observable, runInAction, onBecomeObserved, onBecomeUnobserved } from "../../render-utils/mobxTyped";
3
- import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config, BulkFileInfoListing } from "./BulkDatabaseBase";
3
+ import { BulkDatabaseBase, ReactiveDeps, BulkDatabase2Config, BulkFileInfoListing, MergeAttemptResult } from "./BulkDatabaseBase";
4
4
 
5
5
  export { BulkDatabaseBase, noopReactiveDeps, bulkDatabase2Timing } from "./BulkDatabaseBase";
6
- export type { ReactiveDeps, StorageFactory, BulkDatabase2Config, BulkFileDetails, BulkFileEntry, BulkFileInfoListing } from "./BulkDatabaseBase";
6
+ export type { ReactiveDeps, StorageFactory, BulkDatabase2Config, BulkFileDetails, BulkFileEntry, BulkFileInfoListing, MergeAttemptResult, MergeSkipReason } from "./BulkDatabaseBase";
7
7
 
8
8
  /** Per-column on-disk size info, as reported by getColumnInfo/getReaderInfo. */
9
9
  export type BulkColumnInfo = { column: string; byteSize: number };
@@ -106,8 +106,13 @@ export interface IBulkDatabase2<T extends { key: string }> {
106
106
  */
107
107
  getFileInfo(): Promise<BulkFileInfoListing>;
108
108
 
109
- /** Consolidate on-disk files. Optional to call; the database also does this in the background. */
110
- compact(): Promise<void>;
109
+ /**
110
+ * Consolidate on-disk files. Optional to call; the database also does this in the background.
111
+ * Returns whether anything was merged, or (via skipReason) why the pass never ran — another merge
112
+ * in flight, another tab/process holding the merge lock (with who holds it and when the lock
113
+ * expires), or nothing on disk to compact.
114
+ */
115
+ compact(): Promise<MergeAttemptResult>;
111
116
 
112
117
  /**
113
118
  * Whether this collection's storage is served over the network (a remote server) rather than local
@@ -128,10 +133,11 @@ export interface IBulkDatabase2<T extends { key: string }> {
128
133
  /**
129
134
  * Run one merge pass now (the same policy the database runs on a timer): consolidate recent
130
135
  * fragmentation and dedup a key range if it's worth it. Returns whether it merged anything and
131
- * whether it bailed because another tab/process holds the merge lock so a scheduler can call this
132
- * (e.g. every 30 minutes) and tell "nothing to do" from "someone else is already merging".
136
+ * whether it bailed because another tab/process holds the merge lock (including the lock's holder
137
+ * and expiry) — so a scheduler can call this (e.g. every 30 minutes) and tell "nothing to do" from
138
+ * "someone else is already merging".
133
139
  */
134
- tryMergeNow(): Promise<void>;
140
+ tryMergeNow(): Promise<MergeAttemptResult>;
135
141
 
136
142
  /** Rewrite everything written in [timeLo, timeHi] into fresh key-sorted bulk file(s). Low-level;
137
143
  * most callers want compact() or tryMergeNow(). */
@@ -27,6 +27,13 @@ export type StorageFactory = (path: string) => Promise<FileStorage>;
27
27
  export type BulkDatabase2Config = {
28
28
  maxTriggerThrottleMs?: number;
29
29
  };
30
+ export type MergeSkipReason = "mergeInFlight" | "tabLockHeld" | "fileLockHeld" | "nothingToMerge";
31
+ export type MergeAttemptResult = {
32
+ merged: boolean;
33
+ skipReason?: MergeSkipReason;
34
+ lockHolderId?: string;
35
+ lockExpiresInMs?: number;
36
+ };
30
37
  export declare class BulkDatabaseBase<T extends {
31
38
  key: string;
32
39
  }> {
@@ -50,6 +57,7 @@ export declare class BulkDatabaseBase<T extends {
50
57
  private currentStreamFileName;
51
58
  private currentStreamFileBytes;
52
59
  private mergeInFlight;
60
+ private lastMergeSkipLogMs;
53
61
  private streamRowsOnDisk;
54
62
  private streamBytesOnDisk;
55
63
  private fileSetPollTimer;
@@ -100,8 +108,11 @@ export declare class BulkDatabaseBase<T extends {
100
108
  private processMarkers;
101
109
  private writeBulkFile;
102
110
  private maybeMerge;
103
- tryMergeNow: () => Promise<void>;
104
- compact(): Promise<void>;
111
+ private mergeSkip;
112
+ private runLockedMerge;
113
+ private tryMergeThrottled;
114
+ tryMergeNow(): Promise<MergeAttemptResult>;
115
+ compact(): Promise<MergeAttemptResult>;
105
116
  merge(timeLo: number, timeHi: number): Promise<void>;
106
117
  private readBulkHeader;
107
118
  private fileLogicalSize;
@@ -18,7 +18,7 @@ import { blue, magenta } from "socket-function/src/formatting/logColors";
18
18
  import { STREAM_EXTENSION, frameDeletes, frameRows, streamReaderFromEntries } from "./streamLog";
19
19
  import { broadcast as syncBroadcast, broadcastSeal as syncBroadcastSeal, connect as syncConnect, isSyncSupported, RemoteWrite } from "./syncClient";
20
20
  import { DELETED } from "./WriteOverlay";
21
- import { releaseMergeFileLock, releaseMergeLock, startMergeFileLockHeartbeat, tryAcquireMergeFileLock, tryAcquireMergeLock } from "./mergeLock";
21
+ import { MergeLockInfo, peekMergeFileLock, peekMergeLock, releaseMergeFileLock, releaseMergeLock, startMergeFileLockHeartbeat, tryAcquireMergeFileLock, tryAcquireMergeLock } from "./mergeLock";
22
22
  import { markerExclusions, processDeleteMarkers, readDeleteMarkers, writeDeleteMarker } from "./mergeMarkers";
23
23
  import {
24
24
  BulkFileInfo,
@@ -49,6 +49,9 @@ const DUP_THRESHOLD = 0.4;
49
49
  const DEDUP_TRIGGER_BYTES = 512 * 1024 * 1024;
50
50
  const DEDUP_TRIGGER_FRACTION = 0.5;
51
51
  const WRITE_FLUSH_FIRST_STEP_MS = 250;
52
+ // Skip logs are throttled per collection: the background write path retries merges about once a second,
53
+ // so an unthrottled log would repeat for the whole duration of another tab's merge.
54
+ const MERGE_SKIP_LOG_INTERVAL_MS = 30 * 1000;
52
55
 
53
56
  export const bulkDatabase2Timing = {
54
57
  streamSealAgeMs: 10 * 60 * 60 * 1000,
@@ -67,7 +70,7 @@ export const bulkDatabase2Timing = {
67
70
  // the whole stream file per write.
68
71
  writeFlushMaxDelayMs: isNode() ? 0 : 15 * 1000,
69
72
  fileSetPollIntervalMs: 30 * 60 * 1000,
70
- memoryFlushHeapBytes: 1500 * 1024 * 1024,
73
+ memoryFlushHeapBytes: 5 * 1024 * 1024 * 1024,
71
74
  memoryFlushMinCollectionBytes: 100 * 1024 * 1024,
72
75
  memoryFlushThrottleMs: 15 * 60 * 1000,
73
76
  };
@@ -104,6 +107,26 @@ export type BulkDatabase2Config = {
104
107
  maxTriggerThrottleMs?: number;
105
108
  };
106
109
 
110
+ export type MergeSkipReason =
111
+ // This instance is already mid-merge.
112
+ | "mergeInFlight"
113
+ // Another same-origin tab holds the localStorage merge lock.
114
+ | "tabLockHeld"
115
+ // Another process holds the cross-process .merge-lock file (or won the settle race for it).
116
+ | "fileLockHeld"
117
+ // compact() only: the collection has no files on disk.
118
+ | "nothingToMerge";
119
+
120
+ // What a compact()/tryMergeNow() call did. skipReason is set when the pass never ran; for the lock
121
+ // reasons, lockHolderId/lockExpiresInMs report who holds the lock and how long until it goes stale
122
+ // (so a scheduler knows when retrying could succeed).
123
+ export type MergeAttemptResult = {
124
+ merged: boolean;
125
+ skipReason?: MergeSkipReason;
126
+ lockHolderId?: string;
127
+ lockExpiresInMs?: number;
128
+ };
129
+
107
130
  let networkCompactionEnabled = false;
108
131
 
109
132
  let fileNameCounter = 0;
@@ -237,6 +260,7 @@ export class BulkDatabaseBase<T extends { key: string }> {
237
260
  // tryMergeNow calls) return immediately instead of queueing. Keeps the visibility-timer firings
238
261
  // from stacking behind a long merge.
239
262
  private mergeInFlight = false;
263
+ private lastMergeSkipLogMs = 0;
240
264
  // Running counters of stream-tier rows + bytes on disk. Seeded from each LoadedIndex build, then
241
265
  // incremented per flush so the fold-trigger checks current data without an extra directory listing.
242
266
  private streamRowsOnDisk = 0;
@@ -614,58 +638,77 @@ export class BulkDatabaseBase<T extends { key: string }> {
614
638
  if (!await this.automaticCompactionAllowed()) return;
615
639
  if (opts.onlyIfStreamHeavy && !this.streamNeedsFold()) return;
616
640
  try {
617
- await this.tryMergeNow();
641
+ await this.tryMergeThrottled();
618
642
  } catch (e) {
619
643
  console.warn(`${this.name}: background merge failed: ${(e as Error).message}`);
620
644
  }
621
645
  }
622
646
 
623
- // Returns immediately with { lockFailed: true } if another call is already in flight (same
624
- // instance), if the localStorage lock is held by another tab, or if the cross-process file lock
625
- // says another process owns it. Otherwise: acquires both locks, runs testMerge, releases.
626
- public tryMergeNow = throttleFunction(1000, async () => {
627
- if (this.mergeInFlight) return;
628
- if (!tryAcquireMergeLock(this.name, writerId)) return;
647
+ // Build the "we didn't merge" result and log it (throttled see MERGE_SKIP_LOG_INTERVAL_MS). For
648
+ // lock reasons the log and result include the holder and time until the lock goes stale.
649
+ private async mergeSkip(reason: MergeSkipReason): Promise<MergeAttemptResult> {
650
+ let lock: MergeLockInfo | undefined;
651
+ if (reason === "tabLockHeld") {
652
+ lock = peekMergeLock(this.name);
653
+ } else if (reason === "fileLockHeld") {
654
+ lock = await peekMergeFileLock(await this.storage());
655
+ }
656
+ if (Date.now() - this.lastMergeSkipLogMs >= MERGE_SKIP_LOG_INTERVAL_MS) {
657
+ this.lastMergeSkipLogMs = Date.now();
658
+ const detail = lock && ` (held by ${lock.holderId}, expires in ${formatTime(Math.max(0, lock.expiresInMs))})` || "";
659
+ console.log(`${blue(this.name)} ${magenta("merge")} skipped: ${reason}${detail}`);
660
+ }
661
+ return { merged: false, skipReason: reason, lockHolderId: lock?.holderId, lockExpiresInMs: lock?.expiresInMs };
662
+ }
663
+
664
+ // Acquire both merge locks (same-tab mergeInFlight, cross-tab localStorage, cross-process file),
665
+ // run the pass, release. Skips (with the reason) instead of waiting when any of them is held.
666
+ private async runLockedMerge(run: () => Promise<MergeAttemptResult>): Promise<MergeAttemptResult> {
667
+ if (this.mergeInFlight) return await this.mergeSkip("mergeInFlight");
668
+ if (!tryAcquireMergeLock(this.name, writerId)) return await this.mergeSkip("tabLockHeld");
629
669
  const storage = await this.storage();
630
670
  const haveFileLock = await tryAcquireMergeFileLock(storage, writerId);
631
671
  if (!haveFileLock) {
632
672
  releaseMergeLock(this.name, writerId);
633
- return;
673
+ return await this.mergeSkip("fileLockHeld");
634
674
  }
635
675
  this.mergeInFlight = true;
636
676
  const stopHeartbeat = startMergeFileLockHeartbeat(storage, writerId);
637
677
  try {
638
- await this.testMergeINTERNAL_DO_NOT_CALL();
678
+ return await run();
639
679
  } finally {
640
680
  stopHeartbeat();
641
681
  await releaseMergeFileLock(storage, writerId);
642
682
  releaseMergeLock(this.name, writerId);
643
683
  this.mergeInFlight = false;
644
684
  }
685
+ }
686
+
687
+ // The background path (maybeMerge) goes through this so write bursts coalesce; explicit callers use
688
+ // tryMergeNow directly and get the per-call result.
689
+ private tryMergeThrottled = throttleFunction(1000, async () => {
690
+ await this.tryMergeNow();
645
691
  });
646
692
 
647
- public async compact(): Promise<void> {
648
- if (this.mergeInFlight) return;
649
- if (!tryAcquireMergeLock(this.name, writerId)) return;
650
- const storage = await this.storage();
651
- const haveFileLock = await tryAcquireMergeFileLock(storage, writerId);
652
- if (!haveFileLock) { releaseMergeLock(this.name, writerId); return; }
653
- this.mergeInFlight = true;
654
- const stopHeartbeat = startMergeFileLockHeartbeat(storage, writerId);
655
- try {
693
+ public async tryMergeNow(): Promise<MergeAttemptResult> {
694
+ return await this.runLockedMerge(async () => {
695
+ const merged = await this.testMergeINTERNAL_DO_NOT_CALL();
696
+ return { merged };
697
+ });
698
+ }
699
+
700
+ public async compact(): Promise<MergeAttemptResult> {
701
+ return await this.runLockedMerge(async () => {
656
702
  await this.flushPending();
657
703
  syncBroadcastSeal(this.name);
658
704
  this.streamFileName = undefined;
659
705
  const { bulkFiles, streamFiles } = await this.listFiles();
706
+ if (bulkFiles.length + streamFiles.length < 1) return await this.mergeSkip("nothingToMerge");
660
707
  // compact() folds every file → no older data survives outside it → surviving tombstones
661
708
  // can be dropped (nothing left to suppress).
662
- if (bulkFiles.length + streamFiles.length >= 1) await this.mergeFileSet(bulkFiles, streamFiles, true);
663
- } finally {
664
- stopHeartbeat();
665
- await releaseMergeFileLock(storage, writerId);
666
- releaseMergeLock(this.name, writerId);
667
- this.mergeInFlight = false;
668
- }
709
+ const merged = await this.mergeFileSet(bulkFiles, streamFiles, true);
710
+ return { merged };
711
+ });
669
712
  }
670
713
 
671
714
  public async merge(timeLo: number, timeHi: number): Promise<void> {
@@ -1,6 +1,12 @@
1
1
  import type { FileStorage } from "../FileFolderAPI";
2
+ export type MergeLockInfo = {
3
+ holderId: string;
4
+ expiresInMs: number;
5
+ };
2
6
  export declare function tryAcquireMergeLock(collection: string, holderId: string): boolean;
7
+ export declare function peekMergeLock(collection: string): MergeLockInfo | undefined;
3
8
  export declare function releaseMergeLock(collection: string, holderId: string): void;
9
+ export declare function peekMergeFileLock(storage: FileStorage): Promise<MergeLockInfo | undefined>;
4
10
  export declare function tryAcquireMergeFileLock(storage: FileStorage, holderId: string): Promise<boolean>;
5
11
  export declare function startMergeFileLockHeartbeat(storage: FileStorage, holderId: string): () => void;
6
12
  export declare function releaseMergeFileLock(storage: FileStorage, holderId: string): Promise<void>;
@@ -17,6 +17,16 @@ import type { FileStorage } from "../FileFolderAPI";
17
17
 
18
18
  const LOCK_TTL_MS = 30 * 1000;
19
19
 
20
+ export type MergeLockInfo = { holderId: string; expiresInMs: number };
21
+
22
+ function parseLockToken(token: string): { holderId: string; time: number } | undefined {
23
+ const colon = token.lastIndexOf(":");
24
+ if (colon < 0) return undefined;
25
+ const time = parseInt(token.slice(colon + 1), 10);
26
+ if (!Number.isFinite(time)) return undefined;
27
+ return { holderId: token.slice(0, colon), time };
28
+ }
29
+
20
30
  function getLocalStorage(): Storage | undefined {
21
31
  try {
22
32
  return typeof localStorage !== "undefined" ? localStorage : undefined;
@@ -40,8 +50,8 @@ export function tryAcquireMergeLock(collection: string, holderId: string): boole
40
50
  const now = Date.now();
41
51
  const existing = ls.getItem(key);
42
52
  if (existing && !existing.startsWith(holderId + ":")) {
43
- const t = parseInt(existing.slice(existing.lastIndexOf(":") + 1), 10);
44
- if (Number.isFinite(t) && now - t < LOCK_TTL_MS) return false;
53
+ const parsed = parseLockToken(existing);
54
+ if (parsed && now - parsed.time < LOCK_TTL_MS) return false;
45
55
  }
46
56
  const token = holderId + ":" + now;
47
57
  ls.setItem(key, token);
@@ -49,6 +59,18 @@ export function tryAcquireMergeLock(collection: string, holderId: string): boole
49
59
  return ls.getItem(key) === token;
50
60
  }
51
61
 
62
+ // Non-mutating look at the tab lock: who holds it and how long until it goes stale (negative = already
63
+ // stale). For logging why a merge was skipped.
64
+ export function peekMergeLock(collection: string): MergeLockInfo | undefined {
65
+ const ls = getLocalStorage();
66
+ if (!ls) return undefined;
67
+ const existing = ls.getItem(lockKey(collection));
68
+ if (!existing) return undefined;
69
+ const parsed = parseLockToken(existing);
70
+ if (!parsed) return undefined;
71
+ return { holderId: parsed.holderId, expiresInMs: parsed.time + LOCK_TTL_MS - Date.now() };
72
+ }
73
+
52
74
  export function releaseMergeLock(collection: string, holderId: string): void {
53
75
  const ls = getLocalStorage();
54
76
  if (!ls) return;
@@ -63,19 +85,21 @@ const FILE_LOCK_TTL_MS = 5 * 60 * 1000;
63
85
  const FILE_LOCK_SETTLE_MS = 15 * 1000;
64
86
  const FILE_LOCK_HEARTBEAT_MS = 2 * 60 * 1000;
65
87
 
66
- async function readMergeFileLock(storage: FileStorage): Promise<{ id: string; time: number } | undefined> {
88
+ async function readMergeFileLock(storage: FileStorage): Promise<{ holderId: string; time: number } | undefined> {
67
89
  try {
68
90
  const buf = await storage.get(FILE_LOCK_KEY);
69
91
  if (!buf) return undefined;
70
- const s = buf.toString("utf8");
71
- const colon = s.lastIndexOf(":");
72
- if (colon < 0) return undefined;
73
- const time = parseInt(s.slice(colon + 1), 10);
74
- if (!Number.isFinite(time)) return undefined;
75
- return { id: s.slice(0, colon), time };
92
+ return parseLockToken(buf.toString("utf8"));
76
93
  } catch { return undefined; }
77
94
  }
78
95
 
96
+ // Non-mutating look at the cross-process lock file (see peekMergeLock).
97
+ export async function peekMergeFileLock(storage: FileStorage): Promise<MergeLockInfo | undefined> {
98
+ const existing = await readMergeFileLock(storage);
99
+ if (!existing) return undefined;
100
+ return { holderId: existing.holderId, expiresInMs: existing.time + FILE_LOCK_TTL_MS - Date.now() };
101
+ }
102
+
79
103
  async function writeMergeFileLock(storage: FileStorage, holderId: string, time: number): Promise<void> {
80
104
  await storage.set(FILE_LOCK_KEY, Buffer.from(`${holderId}:${time}`, "utf8") as Buffer);
81
105
  }
@@ -86,11 +110,11 @@ async function writeMergeFileLock(storage: FileStorage, holderId: string, time:
86
110
  export async function tryAcquireMergeFileLock(storage: FileStorage, holderId: string): Promise<boolean> {
87
111
  const now = Date.now();
88
112
  const existing = await readMergeFileLock(storage);
89
- if (existing && existing.id !== holderId && now - existing.time < FILE_LOCK_TTL_MS) return false;
113
+ if (existing && existing.holderId !== holderId && now - existing.time < FILE_LOCK_TTL_MS) return false;
90
114
  await writeMergeFileLock(storage, holderId, now);
91
115
  await new Promise(r => setTimeout(r, FILE_LOCK_SETTLE_MS));
92
116
  const after = await readMergeFileLock(storage);
93
- return !!after && after.id === holderId;
117
+ return !!after && after.holderId === holderId;
94
118
  }
95
119
 
96
120
  // Keep the lock fresh. The merge might run longer than FILE_LOCK_TTL_MS; without periodic re-stamping,
@@ -107,6 +131,6 @@ export function startMergeFileLockHeartbeat(storage: FileStorage, holderId: stri
107
131
  export async function releaseMergeFileLock(storage: FileStorage, holderId: string): Promise<void> {
108
132
  try {
109
133
  const existing = await readMergeFileLock(storage);
110
- if (existing && existing.id === holderId) await storage.remove(FILE_LOCK_KEY);
134
+ if (existing && existing.holderId === holderId) await storage.remove(FILE_LOCK_KEY);
111
135
  } catch { /* ignore */ }
112
136
  }
@@ -627,8 +627,13 @@ export async function autocompactBulkDatabases(root: string): Promise<void> {
627
627
  const start = Date.now();
628
628
  console.log(` [autocompact] ${at} ${name}: starting merge`);
629
629
  try {
630
- await getCompactor(baseDir, name).tryMergeNow();
631
- console.log(` [autocompact] ${at} ${name}: done (${Date.now() - start}ms)`);
630
+ const result = await getCompactor(baseDir, name).tryMergeNow();
631
+ let outcome = result.merged && "merged" || "nothing to merge";
632
+ if (result.skipReason) {
633
+ outcome = `skipped: ${result.skipReason}`;
634
+ if (result.lockExpiresInMs !== undefined) outcome += ` (lock held by ${result.lockHolderId}, expires in ${Math.max(0, Math.round(result.lockExpiresInMs / 1000))}s)`;
635
+ }
636
+ console.log(` [autocompact] ${at} ${name}: done (${outcome}, ${Date.now() - start}ms)`);
632
637
  } catch (e) {
633
638
  console.warn(` [autocompact] ${at} ${name}: failed after ${Date.now() - start}ms - ${(e as Error).message}`);
634
639
  }
package/yarn.lock CHANGED
@@ -1944,10 +1944,10 @@ slash@^3.0.0:
1944
1944
  resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634"
1945
1945
  integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==
1946
1946
 
1947
- socket-function@^1.1.42:
1948
- version "1.1.47"
1949
- resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.1.47.tgz#45cb508b4f402ae37c3053eb8b7cabe1cefaa011"
1950
- integrity sha512-DT+TWWcr6GPQ6E/f8o3p+3UZlHfgnLPCJ6baSY+7jOOm5EdoPXcaCF5L4j4cna4OV4Tpw3ny85b4BqvixXP4sQ==
1947
+ socket-function@^1.1.48:
1948
+ version "1.1.48"
1949
+ resolved "https://registry.yarnpkg.com/socket-function/-/socket-function-1.1.48.tgz#26266a635847497e726ca49061bc604b4ba0abb1"
1950
+ integrity sha512-kKniLUzh3ZneNcNLFxQiiR1TCSAGeyJgHs5S0643s9SmaVZCyEdYTKYgV+u7jzDSjz81wTQVdv/DChVOQNNH3w==
1951
1951
  dependencies:
1952
1952
  "@types/pako" "^2.0.3"
1953
1953
  "@types/ws" "^8.5.3"