pi-cache-optimizer 2.8.26 → 2.8.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.ts +103 -14
- package/package.json +1 -1
- package/src/stats-store.ts +110 -50
package/index.ts
CHANGED
|
@@ -13,7 +13,7 @@ import { atomicCreateTextFileNoReplace, atomicReplaceTextFilePreservingMode, ato
|
|
|
13
13
|
import { getAssistantMessageModelTokenValues, getModelIdNameTokenValues, isAdaptiveGenerationModel, isKimiCodingAdaptiveModel } from "./src/model-detect.ts";
|
|
14
14
|
import { findLastExactModelDefinition, mergeCacheCompat, resolveEffectiveCompatFromConfig } from "./src/compat-config.ts";
|
|
15
15
|
import { analyzeModelsJsonForMissingEntry, applyModelsJsonFixTransaction, chooseFixPlacement, composeFixInsertion, composeMissingEntryInsertion, composeModelOverrideInsertion, composeModelsJsonReceiptRollback, composeProviderAffinityInsertion, createModelsJsonFixReceipt, createRollbackBackupPath, decideFixPlacement, formatCompatKeysForInsertion, hasExplicitLongRetentionOptInFromConfig, hasReceiptReasoningProtocolChange, isActionableModelsJsonFixReceipt, locateModelInJsonc, markModelsJsonFixReceiptRolledBack, parseModelsJsonFixReceipt, prepareModelsJsonRollback, readModelsJsonFixReceipt, readModelsJsonFixReceiptSnapshot, receiptBackupPath, resolveExplicitCompatValue, selfCheckFix, selfCheckMissingEntryInsertion, validateModelsJsonRollback, writeModelsJsonFixReceipt } from "./src/models-json-fix.ts";
|
|
16
|
-
import { type CacheProviderId, type CacheStats, LEGACY_STATE_FILE_PATH, type PersistedRoutedModelRef, type PersistedStatsShardV7, SHARD_FILES_DIR, SHARD_GLOBAL_EPOCH_PATH, SHARD_STATE_DIR, STATE_FILE_PATH, type ShardAggregate, type UsageSnapshot, addUsageToCacheStats, advanceGlobalStatsEpoch, advanceModelStatsEpoch, aggregateStatsShardsV7, cleanupStatsShardsV7, cloneCacheStats, currentLocalDay, emptyAllCacheStats, emptyCacheStats, initialEpoch, loadStatsShardAggregateV7, maybeCleanupStatsShardsV7, modelEpochPath, parseCacheStats, parsePersistedRoutedModelRef, parsePersistedStatsShardV7, readGlobalStatsEpoch, readModelStatsEpoch, readValidStatsShardsV7, removeLegacyStatsFiles, writeStatsShardV7 } from "./src/stats-store.ts";
|
|
16
|
+
import { type CacheProviderId, type CacheStats, LEGACY_STATE_FILE_PATH, type PersistedRoutedModelRef, type PersistedStatsShardV7, SHARD_FILES_DIR, SHARD_GLOBAL_EPOCH_PATH, SHARD_STATE_DIR, STATE_FILE_PATH, type ShardAggregate, type UsageSnapshot, addUsageToCacheStats, advanceGlobalStatsEpoch, advanceModelStatsEpoch, aggregateStatsShardsV7, cleanupStatsShardsV7, cloneCacheStats, mapWithConcurrency, currentLocalDay, emptyAllCacheStats, emptyCacheStats, initialEpoch, loadStatsShardAggregateV7, maybeCleanupStatsShardsV7, modelEpochPath, parseCacheStats, parsePersistedRoutedModelRef, parsePersistedStatsShardV7, readGlobalStatsEpoch, readModelStatsEpoch, readValidStatsShardsV7, removeLegacyStatsFiles, writeStatsShardV7 } from "./src/stats-store.ts";
|
|
17
17
|
import { PI_VIRTUAL_MODEL_API, getCompat, isAnthropicMessagesApi, isKimiCodingEmptySignatureModel, isKnownThirdPartyOpenAIEndpoint, isMistralConversationsApi, isNativeVirtualModel, isOfficialOpenAIBaseUrl, isOpenAICompatibleApi, isOpenAICompatibleProxyApi, isOpenAIFamilyAssistantMessage, isOpenAIFamilyModel, isOpenAIFamilyToken, isPiBuiltInLlamaCppModel, isResponsesPromptRewriteBypassApi, isRoutedFallbackModel, isValidModelsConfigForEffectiveCompat, modelKey, readEffectiveCompatConfig } from "./src/model-identity.ts";
|
|
18
18
|
import { CONFIG_FILE_PATH, VIRTUAL_REWRITE_ENV, FOOTER_MODE_ENV, TOOL_ORDER_ENV, featureEnabled, footerStatsMode, isEnabledEnv, isToolOrderEnabled, parseFooterStatsMode, parsePersistedCacheOptimizerConfig, readPersistedCacheOptimizerConfig, resolveFooterStatsMode, runtimeOptimizerEnabled, setPersistedCacheOptimizerConfig, shouldInjectOpenAIPromptCacheKey, writePersistedCacheOptimizerConfig, writePersistedFeature, writePersistedFooterMode } from "./src/config.ts";
|
|
19
19
|
import { decideZeroPriceWarming, readCacheWarmingMode } from "./src/cache-warming.ts";
|
|
@@ -103,7 +103,13 @@ function notifyCacheCompatIfNeeded(
|
|
|
103
103
|
// Internal helpers exported only so the task verification script
|
|
104
104
|
// (.trellis/tasks/.../verify.ts) can exercise them. They are not part of the
|
|
105
105
|
// extension's public API; pi only invokes the default export below.
|
|
106
|
+
// Most recent background stats restore started by any extension instance in
|
|
107
|
+
// this module. Tests await it after session_start; runtime code uses the
|
|
108
|
+
// per-instance promise instead.
|
|
109
|
+
let latestStatsRestoreForTests: Promise<void> = Promise.resolve();
|
|
110
|
+
|
|
106
111
|
export const __internals_for_tests = {
|
|
112
|
+
waitForStatsRestore: (): Promise<void> => latestStatsRestoreForTests,
|
|
107
113
|
getLastSkillCompressionOutcome,
|
|
108
114
|
decidePromptCacheRetention,
|
|
109
115
|
userRequestedLongCacheRetention,
|
|
@@ -317,6 +323,7 @@ export const __internals_for_tests = {
|
|
|
317
323
|
aggregateStatsShardsV7,
|
|
318
324
|
loadStatsShardAggregateV7,
|
|
319
325
|
cleanupStatsShardsV7,
|
|
326
|
+
mapWithConcurrency,
|
|
320
327
|
removeLegacyStatsFiles,
|
|
321
328
|
readGlobalStatsEpoch,
|
|
322
329
|
readModelStatsEpoch,
|
|
@@ -410,6 +417,20 @@ export default function (pi: ExtensionAPI) {
|
|
|
410
417
|
let persistTimer: ReturnType<typeof setTimeout> | null = null;
|
|
411
418
|
let shardRefreshTimer: ReturnType<typeof setTimeout> | null = null;
|
|
412
419
|
let shardWatcher: ReturnType<typeof watch> | undefined;
|
|
420
|
+
// Historical stats are restored in the background so a large shard
|
|
421
|
+
// directory never holds up Pi's session_start (issue #28). Handlers that
|
|
422
|
+
// read or persist aggregated stats await this first.
|
|
423
|
+
let statsRestorePromise: Promise<void> = Promise.resolve();
|
|
424
|
+
let statsRestoreGeneration = 0;
|
|
425
|
+
async function waitForStatsRestore(): Promise<void> {
|
|
426
|
+
// A session_start may chain a newer restore while we wait; keep waiting
|
|
427
|
+
// until the chain is stable so callers never act on a half-restored view.
|
|
428
|
+
let current: Promise<void>;
|
|
429
|
+
do {
|
|
430
|
+
current = statsRestorePromise;
|
|
431
|
+
await current;
|
|
432
|
+
} while (current !== statsRestorePromise);
|
|
433
|
+
}
|
|
413
434
|
const enqueuePersist = createSerializedAsyncRunner();
|
|
414
435
|
let instanceId = randomUUID();
|
|
415
436
|
const instanceStartedAt = Date.now();
|
|
@@ -664,7 +685,8 @@ export default function (pi: ExtensionAPI) {
|
|
|
664
685
|
if (shardRefreshTimer !== null) clearTimeout(shardRefreshTimer);
|
|
665
686
|
shardRefreshTimer = setTimeout(() => {
|
|
666
687
|
shardRefreshTimer = null;
|
|
667
|
-
void
|
|
688
|
+
void waitForStatsRestore()
|
|
689
|
+
.then(() => refreshShardAggregate())
|
|
668
690
|
.then(() => publishStatus(ctx as ExtensionContext, model ?? ctx?.model))
|
|
669
691
|
.catch(() => undefined);
|
|
670
692
|
}, SHARD_REFRESH_DEBOUNCE_MS);
|
|
@@ -771,7 +793,12 @@ export default function (pi: ExtensionAPI) {
|
|
|
771
793
|
if (persistTimer !== null) clearTimeout(persistTimer);
|
|
772
794
|
persistTimer = setTimeout(() => {
|
|
773
795
|
persistTimer = null;
|
|
774
|
-
|
|
796
|
+
// A persist scheduled while a background restore runs (e.g. a routed
|
|
797
|
+
// model change without usage) must not race the restore's own write.
|
|
798
|
+
void waitForStatsRestore()
|
|
799
|
+
.then(() => persistCacheStats(ctx))
|
|
800
|
+
.then(() => refreshShardAggregate())
|
|
801
|
+
.catch(() => undefined);
|
|
775
802
|
}, PERSIST_DEBOUNCE_MS);
|
|
776
803
|
persistTimer.unref?.();
|
|
777
804
|
}
|
|
@@ -818,7 +845,12 @@ export default function (pi: ExtensionAPI) {
|
|
|
818
845
|
finally { rolloverPromise = undefined; }
|
|
819
846
|
}
|
|
820
847
|
|
|
821
|
-
|
|
848
|
+
/**
|
|
849
|
+
* Synchronous part of a session restore: reset in-memory state immediately
|
|
850
|
+
* at session_start, so usage recorded before the background disk restore
|
|
851
|
+
* finishes is never wiped by a reset that runs later.
|
|
852
|
+
*/
|
|
853
|
+
function resetStatsForSessionStart(reason: string, ctx: ExtensionContext): void {
|
|
822
854
|
syncSessionHash(ctx);
|
|
823
855
|
lastStatusText = undefined;
|
|
824
856
|
cacheStatsProcessByModel = {};
|
|
@@ -826,19 +858,48 @@ export default function (pi: ExtensionAPI) {
|
|
|
826
858
|
modelEpochByKey.clear();
|
|
827
859
|
modelApiByKey.clear();
|
|
828
860
|
modelNameByKey.clear();
|
|
829
|
-
currentGlobalEpoch = await readGlobalStatsEpoch();
|
|
830
861
|
if (reason === "reload") {
|
|
831
862
|
clearRecentSamples();
|
|
832
863
|
}
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
async function restoreCacheStats(ctx: ExtensionContext): Promise<void> {
|
|
867
|
+
// Session identity was synced by resetStatsForSessionStart. Usage may
|
|
868
|
+
// already have been recorded (message_end does not wait to record), and
|
|
869
|
+
// message_end syncs the epoch before recording. If the disk epoch differs
|
|
870
|
+
// now, a reset happened after that usage: adopt it and drop the stale
|
|
871
|
+
// process bucket instead of re-labelling old counters with the new epoch.
|
|
872
|
+
const latestGlobalEpoch = await readGlobalStatsEpoch();
|
|
873
|
+
if (latestGlobalEpoch !== currentGlobalEpoch) {
|
|
874
|
+
currentGlobalEpoch = latestGlobalEpoch;
|
|
875
|
+
cacheStatsProcessByModel = {};
|
|
876
|
+
modelEpochByKey.clear();
|
|
877
|
+
modelApiByKey.clear();
|
|
878
|
+
modelNameByKey.clear();
|
|
879
|
+
}
|
|
833
880
|
await removeLegacyStatsFiles();
|
|
834
881
|
await mkdir(SHARD_FILES_DIR, { recursive: true });
|
|
882
|
+
await rollOverStatsIfNeeded(ctx);
|
|
883
|
+
// Run due retention maintenance before the single startup scan, so the
|
|
884
|
+
// aggregate never retains counters from files maintenance just removed and
|
|
885
|
+
// no second directory pass is needed. Cleanup also warms the parsed-shard
|
|
886
|
+
// cache, so the scan below only re-stats files.
|
|
887
|
+
try {
|
|
888
|
+
await maybeCleanupStatsShardsV7();
|
|
889
|
+
} catch (error) {
|
|
890
|
+
console.warn(`${LOG_PREFIX}: stats shard maintenance failed`, error);
|
|
891
|
+
}
|
|
892
|
+
// One scan adopts other processes' epochs and restores this session's
|
|
893
|
+
// routed-model metadata before this instance publishes its own shard.
|
|
894
|
+
// In-memory process counters were cleared above, so no pre-persist
|
|
895
|
+
// re-scan is needed; this instance's own shard is always replaced by its
|
|
896
|
+
// in-memory copy during aggregation, so no post-persist scan either.
|
|
835
897
|
await refreshShardAggregate();
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
await refreshShardAggregate();
|
|
898
|
+
if (persistTimer !== null) {
|
|
899
|
+
clearTimeout(persistTimer);
|
|
900
|
+
persistTimer = null;
|
|
901
|
+
}
|
|
902
|
+
await persistCacheStats(ctx);
|
|
842
903
|
}
|
|
843
904
|
|
|
844
905
|
async function publishStatus(ctx: ExtensionContext, model: PiModel | undefined = ctx.model): Promise<void> {
|
|
@@ -968,7 +1029,26 @@ export default function (pi: ExtensionAPI) {
|
|
|
968
1029
|
|
|
969
1030
|
pi.on("session_start", async (event, ctx) => {
|
|
970
1031
|
if (runtimeOptimizerEnabled) requestLongCacheRetention();
|
|
971
|
-
|
|
1032
|
+
// Do not block Pi's startup on historical stats (issue #28). The
|
|
1033
|
+
// restore runs in the background (chained after any earlier restore so
|
|
1034
|
+
// two never interleave) and the footer is published when it finishes.
|
|
1035
|
+
// A newer session_start supersedes an older restore, which then must not
|
|
1036
|
+
// publish a stale footer. Handlers that read or persist stats await
|
|
1037
|
+
// waitForStatsRestore() first.
|
|
1038
|
+
const generation = ++statsRestoreGeneration;
|
|
1039
|
+
resetStatsForSessionStart(event.reason, ctx);
|
|
1040
|
+
const previousRestore = statsRestorePromise;
|
|
1041
|
+
statsRestorePromise = previousRestore
|
|
1042
|
+
// A restore superseded before it started is skipped entirely: the newer
|
|
1043
|
+
// one redoes all of its work, and its ctx may already be stale.
|
|
1044
|
+
.then(() => (generation === statsRestoreGeneration ? restoreCacheStats(ctx) : undefined))
|
|
1045
|
+
.then(async () => {
|
|
1046
|
+
if (generation === statsRestoreGeneration) await publishStatus(ctx);
|
|
1047
|
+
})
|
|
1048
|
+
.catch((error) => {
|
|
1049
|
+
console.warn(`${LOG_PREFIX}: failed to restore cache stats`, error);
|
|
1050
|
+
});
|
|
1051
|
+
latestStatsRestoreForTests = statsRestorePromise;
|
|
972
1052
|
if (ctx.mode === "tui" && !shardWatcher) {
|
|
973
1053
|
try {
|
|
974
1054
|
shardWatcher = watch(SHARD_FILES_DIR, () => scheduleShardRefresh(ctx));
|
|
@@ -977,13 +1057,13 @@ export default function (pi: ExtensionAPI) {
|
|
|
977
1057
|
// Lifecycle and explicit command refreshes remain authoritative.
|
|
978
1058
|
}
|
|
979
1059
|
}
|
|
980
|
-
await publishStatus(ctx);
|
|
981
1060
|
});
|
|
982
1061
|
|
|
983
1062
|
pi.on("tool_execution_end", async (event, ctx) => {
|
|
984
1063
|
// Pi 0.99+ codemode scripts run nested tool calls in parallel; the calling
|
|
985
1064
|
// tool's own end event follows them, so one refresh covers the batch.
|
|
986
1065
|
if (isNonEmptyString((event as { parentToolCallId?: unknown } | undefined)?.parentToolCallId)) return;
|
|
1066
|
+
await waitForStatsRestore();
|
|
987
1067
|
await refreshShardAggregate();
|
|
988
1068
|
await publishStatus(ctx);
|
|
989
1069
|
});
|
|
@@ -1003,12 +1083,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
1003
1083
|
// a record consumed out of order) so they cannot be correlated with the
|
|
1004
1084
|
// next run's messages.
|
|
1005
1085
|
providerRequestStates.length = 0;
|
|
1086
|
+
await waitForStatsRestore();
|
|
1006
1087
|
await refreshShardAggregate();
|
|
1007
1088
|
await publishStatus(ctx);
|
|
1008
1089
|
});
|
|
1009
1090
|
|
|
1010
1091
|
pi.on("session_shutdown", async (_event, ctx) => {
|
|
1011
1092
|
try {
|
|
1093
|
+
await waitForStatsRestore();
|
|
1012
1094
|
await flushPersistCacheStats(ctx, "closed");
|
|
1013
1095
|
} finally {
|
|
1014
1096
|
if (shardRefreshTimer !== null) clearTimeout(shardRefreshTimer);
|
|
@@ -1025,6 +1107,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
1025
1107
|
|
|
1026
1108
|
pi.on("model_select", async (event, ctx) => {
|
|
1027
1109
|
if (runtimeOptimizerEnabled) notifyCacheCompatIfNeeded(resolveRouteModel(event.model, ctx) ?? event.model, ctx, warnedModels);
|
|
1110
|
+
await waitForStatsRestore();
|
|
1028
1111
|
await refreshShardAggregate();
|
|
1029
1112
|
await publishStatus(ctx, event.model);
|
|
1030
1113
|
});
|
|
@@ -1568,6 +1651,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
1568
1651
|
addUsageToCacheStats(getStatsForModel(undefined, adapter), usage);
|
|
1569
1652
|
}
|
|
1570
1653
|
|
|
1654
|
+
await waitForStatsRestore();
|
|
1571
1655
|
schedulePersistCacheStats(ctx);
|
|
1572
1656
|
await refreshShardAggregate();
|
|
1573
1657
|
// A late response from an older selection must not replace the active
|
|
@@ -1627,6 +1711,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
1627
1711
|
pi.registerCommand("cache-optimizer", {
|
|
1628
1712
|
description: "Configure and diagnose Pi cache behavior",
|
|
1629
1713
|
getArgumentCompletions: getCacheOptimizerArgumentCompletions,
|
|
1630
|
-
handler: handleCacheOptimizerCommand
|
|
1714
|
+
handler: async (args: string, cmdCtx: Parameters<typeof handleCacheOptimizerCommand>[1]) => {
|
|
1715
|
+
// Commands read and reset aggregated stats; let a background restore
|
|
1716
|
+
// finish first so they never act on a half-restored view.
|
|
1717
|
+
await waitForStatsRestore();
|
|
1718
|
+
return handleCacheOptimizerCommand(args, cmdCtx);
|
|
1719
|
+
},
|
|
1631
1720
|
});
|
|
1632
1721
|
}
|
package/package.json
CHANGED
package/src/stats-store.ts
CHANGED
|
@@ -345,9 +345,75 @@ type CachedShardRead = {
|
|
|
345
345
|
ino: number | bigint;
|
|
346
346
|
size: number;
|
|
347
347
|
mtimeMs: number;
|
|
348
|
+
/** Valid shard whose embedded instanceId matches its filename. */
|
|
348
349
|
shard: PersistedStatsShardV7 | undefined;
|
|
350
|
+
/** Schema-valid parse regardless of filename match (used by retention). */
|
|
351
|
+
parsed: PersistedStatsShardV7 | undefined;
|
|
349
352
|
};
|
|
350
353
|
|
|
354
|
+
/**
|
|
355
|
+
* Upper bound on concurrent filesystem operations during shard scans. Startup
|
|
356
|
+
* on Windows (antivirus-scanned AppData) was dominated by thousands of
|
|
357
|
+
* serialized lstat/readFile awaits; a small pool hides per-call latency
|
|
358
|
+
* without opening an unbounded number of descriptors.
|
|
359
|
+
*/
|
|
360
|
+
export const SHARD_IO_CONCURRENCY = 16;
|
|
361
|
+
|
|
362
|
+
/** Run `worker` over `items` with at most `limit` in flight; results keep input order. */
|
|
363
|
+
export async function mapWithConcurrency<T, R>(
|
|
364
|
+
items: readonly T[],
|
|
365
|
+
limit: number,
|
|
366
|
+
worker: (item: T, index: number) => Promise<R>,
|
|
367
|
+
): Promise<R[]> {
|
|
368
|
+
const results = new Array<R>(items.length);
|
|
369
|
+
let next = 0;
|
|
370
|
+
const run = async (): Promise<void> => {
|
|
371
|
+
while (next < items.length) {
|
|
372
|
+
const index = next++;
|
|
373
|
+
results[index] = await worker(items[index] as T, index);
|
|
374
|
+
}
|
|
375
|
+
};
|
|
376
|
+
const pool = Math.max(1, Math.min(limit, items.length));
|
|
377
|
+
await Promise.all(Array.from({ length: pool }, run));
|
|
378
|
+
return results;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
type ShardFileRead = {
|
|
382
|
+
info: import("node:fs").Stats;
|
|
383
|
+
/** Valid shard with matching filename instanceId. */
|
|
384
|
+
shard: PersistedStatsShardV7 | undefined;
|
|
385
|
+
/** Schema-valid parse regardless of filename match. */
|
|
386
|
+
parsed: PersistedStatsShardV7 | undefined;
|
|
387
|
+
};
|
|
388
|
+
|
|
389
|
+
/**
|
|
390
|
+
* Stat a shard file and return its parsed contents, re-reading only when the
|
|
391
|
+
* inode/size/mtime changed. Returns undefined for non-regular files. Throws on
|
|
392
|
+
* read/parse failure (after dropping any cache entry) so callers can apply
|
|
393
|
+
* their own policy.
|
|
394
|
+
*/
|
|
395
|
+
async function readShardFileCached(path: string, name: string): Promise<ShardFileRead | undefined> {
|
|
396
|
+
try {
|
|
397
|
+
const info = await lstat(path);
|
|
398
|
+
if (!info.isFile() || info.isSymbolicLink()) {
|
|
399
|
+
shardReadCache.delete(path);
|
|
400
|
+
return undefined;
|
|
401
|
+
}
|
|
402
|
+
const cached = shardReadCache.get(path);
|
|
403
|
+
if (cached && cached.ino === info.ino && cached.size === info.size && cached.mtimeMs === info.mtimeMs) {
|
|
404
|
+
return { info, shard: cached.shard, parsed: cached.parsed };
|
|
405
|
+
}
|
|
406
|
+
const filenameInstanceId = name.slice(0, -".json".length);
|
|
407
|
+
const parsed = parsePersistedStatsShardV7(JSON.parse(await readFile(path, "utf8")));
|
|
408
|
+
const shard = parsed && parsed.instanceId === filenameInstanceId ? parsed : undefined;
|
|
409
|
+
shardReadCache.set(path, { ino: info.ino, size: info.size, mtimeMs: info.mtimeMs, shard, parsed });
|
|
410
|
+
return { info, shard, parsed };
|
|
411
|
+
} catch (error) {
|
|
412
|
+
shardReadCache.delete(path);
|
|
413
|
+
throw error;
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
351
417
|
/**
|
|
352
418
|
* Parsed-shard cache keyed by path and validated by inode, size, and mtime.
|
|
353
419
|
* Writers always publish through temp-file + rename, so an updated shard gets
|
|
@@ -371,37 +437,22 @@ export async function readValidStatsShardsV7(directory: string = SHARD_FILES_DIR
|
|
|
371
437
|
if (getErrorCode(error) === "ENOENT") return [];
|
|
372
438
|
throw error;
|
|
373
439
|
}
|
|
374
|
-
const shards: PersistedStatsShardV7[] = [];
|
|
375
440
|
const seenPaths = new Set<string>();
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
seenPaths.add(path);
|
|
441
|
+
const candidates = names.filter((name) => /^[0-9a-f-]+\.json$/i.test(name));
|
|
442
|
+
for (const name of candidates) seenPaths.add(join(directory, name));
|
|
443
|
+
const reads = await mapWithConcurrency(candidates, SHARD_IO_CONCURRENCY, async (name) => {
|
|
380
444
|
try {
|
|
381
|
-
|
|
382
|
-
if (!info.isFile() || info.isSymbolicLink()) {
|
|
383
|
-
shardReadCache.delete(path);
|
|
384
|
-
continue;
|
|
385
|
-
}
|
|
386
|
-
const cached = shardReadCache.get(path);
|
|
387
|
-
let parsed: PersistedStatsShardV7 | undefined;
|
|
388
|
-
if (cached && cached.ino === info.ino && cached.size === info.size && cached.mtimeMs === info.mtimeMs) {
|
|
389
|
-
parsed = cached.shard;
|
|
390
|
-
} else {
|
|
391
|
-
const filenameInstanceId = name.slice(0, -".json".length);
|
|
392
|
-
const candidate = parsePersistedStatsShardV7(JSON.parse(await readFile(path, "utf8")));
|
|
393
|
-
parsed = candidate && candidate.instanceId === filenameInstanceId ? candidate : undefined;
|
|
394
|
-
shardReadCache.set(path, { ino: info.ino, size: info.size, mtimeMs: info.mtimeMs, shard: parsed });
|
|
395
|
-
}
|
|
396
|
-
// Callers may mutate the aggregate they build; hand out copies so the
|
|
397
|
-
// cache stays authoritative.
|
|
398
|
-
if (parsed) shards.push(cloneShard(parsed));
|
|
445
|
+
return (await readShardFileCached(join(directory, name), name))?.shard;
|
|
399
446
|
} catch {
|
|
400
447
|
// Ignore malformed or transiently unavailable shards. Atomic writers will
|
|
401
448
|
// publish a complete replacement on the next successful update.
|
|
402
|
-
|
|
449
|
+
return undefined;
|
|
403
450
|
}
|
|
404
|
-
}
|
|
451
|
+
});
|
|
452
|
+
// Callers may mutate the aggregate they build; hand out copies so the cache
|
|
453
|
+
// stays authoritative. Preserve directory order for deterministic ties.
|
|
454
|
+
const shards: PersistedStatsShardV7[] = [];
|
|
455
|
+
for (const shard of reads) if (shard) shards.push(cloneShard(shard));
|
|
405
456
|
for (const path of Array.from(shardReadCache.keys())) {
|
|
406
457
|
if (!seenPaths.has(path) && path.startsWith(directory)) shardReadCache.delete(path);
|
|
407
458
|
}
|
|
@@ -491,67 +542,76 @@ export async function cleanupStatsShardsV7(now = Date.now(), directory: string =
|
|
|
491
542
|
return getErrorCode(error) === "ENOENT" ? 0 : Promise.reject(error);
|
|
492
543
|
}
|
|
493
544
|
const today = currentLocalDay();
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
const
|
|
497
|
-
const isTemp = name.endsWith(".tmp");
|
|
498
|
-
if (!isShard && !isTemp) continue;
|
|
545
|
+
const candidates = names.filter((name) => /^[0-9a-f-]+\.json$/i.test(name) || name.endsWith(".tmp"));
|
|
546
|
+
const outcomes = await mapWithConcurrency(candidates, SHARD_IO_CONCURRENCY, async (name): Promise<number> => {
|
|
547
|
+
const isTemp = !/^[0-9a-f-]+\.json$/i.test(name);
|
|
499
548
|
const path = join(directory, name);
|
|
500
549
|
try {
|
|
501
|
-
const info = await lstat(path);
|
|
502
|
-
if (!info.isFile() || info.isSymbolicLink()) continue;
|
|
503
550
|
if (isTemp) {
|
|
504
|
-
|
|
551
|
+
const info = await lstat(path);
|
|
552
|
+
if (!info.isFile() || info.isSymbolicLink()) return 0;
|
|
553
|
+
if (now - info.mtimeMs < SHARD_TEMP_RETENTION_MS) return 0;
|
|
505
554
|
} else {
|
|
506
|
-
|
|
507
|
-
let
|
|
555
|
+
let read: ShardFileRead | undefined;
|
|
556
|
+
let mtimeMs: number;
|
|
508
557
|
try {
|
|
509
|
-
|
|
510
|
-
|
|
558
|
+
read = await readShardFileCached(path, name);
|
|
559
|
+
if (!read) return 0;
|
|
560
|
+
mtimeMs = read.info.mtimeMs;
|
|
561
|
+
} catch (error) {
|
|
511
562
|
// A corrupt JSON shard has no usable lifecycle metadata. Apply the
|
|
512
563
|
// same mtime retention as schema-invalid shards instead of keeping
|
|
513
564
|
// it forever; read failures still leave the file untouched.
|
|
565
|
+
if (!(error instanceof SyntaxError)) return 0;
|
|
566
|
+
const info = await lstat(path);
|
|
567
|
+
if (!info.isFile() || info.isSymbolicLink()) return 0;
|
|
568
|
+
mtimeMs = info.mtimeMs;
|
|
514
569
|
}
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
if (
|
|
570
|
+
const parsed = read?.parsed;
|
|
571
|
+
if (parsed?.day === today) return 0;
|
|
572
|
+
const updatedAt = parsed?.lifecycle.updatedAt ?? mtimeMs;
|
|
573
|
+
if (now - updatedAt < SHARD_RETENTION_MS) return 0;
|
|
574
|
+
if (parsed?.lifecycle.state === "active" && isProcessAlive(parsed.process.pid)) return 0;
|
|
519
575
|
}
|
|
520
576
|
await unlink(path);
|
|
521
|
-
|
|
577
|
+
shardReadCache.delete(path);
|
|
578
|
+
return 1;
|
|
522
579
|
} catch {
|
|
523
580
|
// Best-effort maintenance must never block the extension.
|
|
581
|
+
return 0;
|
|
524
582
|
}
|
|
525
|
-
}
|
|
526
|
-
return
|
|
583
|
+
});
|
|
584
|
+
return outcomes.reduce((sum, value) => sum + value, 0);
|
|
527
585
|
}
|
|
528
586
|
|
|
529
|
-
|
|
587
|
+
/** Run due retention maintenance; resolves to the number of files removed (0 when skipped). */
|
|
588
|
+
export async function maybeCleanupStatsShardsV7(now = Date.now()): Promise<number> {
|
|
530
589
|
await mkdir(SHARD_MAINTENANCE_DIR, { recursive: true });
|
|
531
590
|
try {
|
|
532
591
|
const marker = await stat(SHARD_CLEANUP_MARKER_PATH);
|
|
533
|
-
if (now - marker.mtimeMs < SHARD_CLEANUP_INTERVAL_MS) return;
|
|
592
|
+
if (now - marker.mtimeMs < SHARD_CLEANUP_INTERVAL_MS) return 0;
|
|
534
593
|
} catch {}
|
|
535
594
|
|
|
536
595
|
try {
|
|
537
596
|
await mkdir(SHARD_CLEANUP_LOCK_PATH);
|
|
538
597
|
} catch (error) {
|
|
539
|
-
if (getErrorCode(error) !== "EEXIST") return;
|
|
598
|
+
if (getErrorCode(error) !== "EEXIST") return 0;
|
|
540
599
|
try {
|
|
541
600
|
const lock = await stat(SHARD_CLEANUP_LOCK_PATH);
|
|
542
|
-
if (now - lock.mtimeMs <= SHARD_CLEANUP_LOCK_STALE_MS) return;
|
|
601
|
+
if (now - lock.mtimeMs <= SHARD_CLEANUP_LOCK_STALE_MS) return 0;
|
|
543
602
|
await rm(SHARD_CLEANUP_LOCK_PATH, { recursive: true, force: true });
|
|
544
603
|
await mkdir(SHARD_CLEANUP_LOCK_PATH);
|
|
545
604
|
} catch {
|
|
546
|
-
return;
|
|
605
|
+
return 0;
|
|
547
606
|
}
|
|
548
607
|
}
|
|
549
608
|
|
|
550
609
|
try {
|
|
551
|
-
await cleanupStatsShardsV7(now);
|
|
610
|
+
const removed = await cleanupStatsShardsV7(now);
|
|
552
611
|
const tempPath = `${SHARD_CLEANUP_MARKER_PATH}.${process.pid}.${randomUUID()}.tmp`;
|
|
553
612
|
await writeFile(tempPath, `${now}\n`, "utf8");
|
|
554
613
|
await rename(tempPath, SHARD_CLEANUP_MARKER_PATH);
|
|
614
|
+
return removed;
|
|
555
615
|
} finally {
|
|
556
616
|
await rm(SHARD_CLEANUP_LOCK_PATH, { recursive: true, force: true });
|
|
557
617
|
}
|