sweet-search 2.5.13 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -9
- package/core/cli.js +41 -3
- package/core/embedding/embedding-local-model.js +106 -10
- package/core/embedding/embedding-service.js +59 -1
- package/core/embedding/model-client.mjs +257 -0
- package/core/embedding/model-server.mjs +217 -0
- package/core/incremental-indexing/application/maintenance-handlers.mjs +19 -98
- package/core/incremental-indexing/application/maintenance-worker.mjs +46 -9
- package/core/incremental-indexing/application/operator-cli.mjs +14 -5
- package/core/incremental-indexing/application/production-reconciler-helpers.mjs +40 -0
- package/core/incremental-indexing/application/production-reconciler.mjs +718 -54
- package/core/incremental-indexing/application/reconciler.mjs +87 -15
- package/core/incremental-indexing/domain/cutoff-cache.mjs +191 -0
- package/core/incremental-indexing/domain/interval-autotune.mjs +84 -1
- package/core/incremental-indexing/domain/reconcile-counters.mjs +0 -4
- package/core/incremental-indexing/domain/watermark-scheduler.mjs +0 -24
- package/core/incremental-indexing/infrastructure/maintenance-state-reader.mjs +2 -26
- package/core/incremental-indexing/infrastructure/manifest.mjs +1 -9
- package/core/incremental-indexing/infrastructure/sqlite-fts5.mjs +72 -0
- package/core/indexing/artifact-builder.js +1 -1
- package/core/indexing/dedup/dedup-phase.js +36 -17
- package/core/indexing/dedup/exemplar-selector.js +5 -0
- package/core/indexing/index-codebase-v21.js +37 -14
- package/core/indexing/index-maintainer.mjs +337 -6
- package/core/indexing/indexer-ann.js +27 -434
- package/core/indexing/indexer-build.js +30 -14
- package/core/indexing/indexer-manifest.js +0 -3
- package/core/indexing/indexer-phases.js +101 -25
- package/core/indexing/maintainer-launcher.mjs +22 -0
- package/core/indexing/maintainer-watcher.mjs +397 -0
- package/core/indexing/os-priority.mjs +160 -0
- package/core/indexing/rss-budget.mjs +425 -0
- package/core/indexing/streaming-vectors.js +450 -0
- package/core/infrastructure/config/platform.js +14 -10
- package/core/infrastructure/onnx-session-utils.js +37 -0
- package/core/infrastructure/sparse-gram-delta-reader.js +11 -1
- package/core/ranking/late-interaction-index.js +58 -7
- package/core/search/daemon-registry.js +199 -0
- package/core/search/search-read-semantic.js +9 -3
- package/core/search/search-semantic.js +6 -29
- package/core/search/search-server.js +527 -27
- package/core/search/session-daemon-prewarm.mjs +110 -1
- package/core/search/sweet-search.js +0 -38
- package/core/vector-store/binary-hnsw-index.js +692 -78
- package/core/vector-store/index.js +1 -4
- package/eval/agent-read-workflows/bin/_ss-argparse.mjs +51 -5
- package/eval/agent-read-workflows/bin/_ss-helpers.mjs +95 -44
- package/eval/agent-read-workflows/bin/ss-read +2 -0
- package/mcp/tool-handlers.js +1 -2
- package/package.json +11 -8
- package/scripts/uninstall.js +2 -0
- package/core/vector-store/hnsw-index.js +0 -751
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Best-effort OS background-priority helper for the maintainer daemon child.
|
|
3
|
+
*
|
|
4
|
+
* Implements research-doc §4.A recommendations A.2 (macOS `taskpolicy -b`) and
|
|
5
|
+
* A.3 (Linux `ionice -c3` + `chrt -b`): once the launcher has spawned the
|
|
6
|
+
* detached maintainer, we ask the OS to treat that *child* as background work
|
|
7
|
+
* (CPU + I/O throttle, E-core routing on Apple Silicon). This is purely a
|
|
8
|
+
* scheduling hint — the indexed output is identical; only *when* the kernel
|
|
9
|
+
* grants CPU/IO to the child changes.
|
|
10
|
+
*
|
|
11
|
+
* Design contract:
|
|
12
|
+
* - `applyBackgroundPriority(pid)` is **best-effort and NEVER throws**. Every
|
|
13
|
+
* spawn is wrapped so a missing binary (`taskpolicy`/`ionice`/`chrt`), an
|
|
14
|
+
* EPERM, or any other error is swallowed silently — demotion is an
|
|
15
|
+
* optimization, not a correctness requirement.
|
|
16
|
+
* - It runs in the *foreground caller's* process (the launcher), targeting
|
|
17
|
+
* the child by pid, so the caller's own priority is never affected.
|
|
18
|
+
* - Unknown platforms are a no-op.
|
|
19
|
+
*
|
|
20
|
+
* The optional native N-API addon (research §4.A A.5/A.6: Windows
|
|
21
|
+
* `PROCESS_MODE_BACKGROUND_BEGIN`, macOS per-thread `QOS_CLASS_BACKGROUND`)
|
|
22
|
+
* slots in via `loadNativePriorityAddon()`. It is published as a per-platform
|
|
23
|
+
* `optionalDependency` (`@sweet-search/bg-priority`), so a failed native build
|
|
24
|
+
* does NOT break `npm install` and the loader simply returns `null` — every
|
|
25
|
+
* code path then falls back to the spawn-based helper below.
|
|
26
|
+
*
|
|
27
|
+
* IMPORTANT semantic difference between the two levers:
|
|
28
|
+
* - `applyBackgroundPriority(pid)` demotes an *arbitrary child* pid and is
|
|
29
|
+
* therefore spawn-based only (`taskpolicy`/`ionice`/`chrt` accept a pid).
|
|
30
|
+
* The native addon CANNOT do this — Windows `SetPriorityClass` /
|
|
31
|
+
* macOS `pthread_set_qos_class_self_np` only affect the *calling*
|
|
32
|
+
* process/thread. So `applyBackgroundPriority` is left unchanged.
|
|
33
|
+
* - `setNativeBackgroundMode(enable)` is the *self*-demotion path: the
|
|
34
|
+
* maintainer daemon calls it on ITSELF, gated on
|
|
35
|
+
* `SWEET_SEARCH_MAINTAINER_NATIVE_PRIORITY==='1'` AND the addon being
|
|
36
|
+
* present. When the addon is absent (default), it returns `false` and the
|
|
37
|
+
* caller keeps its `os.setPriority(PRIORITY_LOW)` / spawn fallback.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
import { spawn } from 'node:child_process';
|
|
41
|
+
import { createRequire } from 'node:module';
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Spawn a helper that demotes `pid`, swallowing every failure mode.
|
|
45
|
+
* Detached + unref'd + stdio:'ignore' so it never holds the caller open and
|
|
46
|
+
* never writes to the caller's stdio. A throw from `spawn` itself (e.g. an
|
|
47
|
+
* exotic platform restriction) is caught; an async spawn `error` (e.g. ENOENT
|
|
48
|
+
* for a missing binary) is caught via the `error` listener.
|
|
49
|
+
*
|
|
50
|
+
* @param {string} command
|
|
51
|
+
* @param {string[]} args
|
|
52
|
+
*/
|
|
53
|
+
function spawnBestEffort(command, args) {
|
|
54
|
+
try {
|
|
55
|
+
const child = spawn(command, args, { stdio: 'ignore', detached: true });
|
|
56
|
+
// A missing binary surfaces asynchronously as an 'error' event, which would
|
|
57
|
+
// otherwise become an unhandled exception that crashes the process. Swallow.
|
|
58
|
+
child.on('error', () => {});
|
|
59
|
+
child.unref();
|
|
60
|
+
} catch {
|
|
61
|
+
/* never throw — best-effort only */
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Demote the given child process to OS background priority. Best-effort,
|
|
67
|
+
* platform-dispatched, NEVER throws.
|
|
68
|
+
*
|
|
69
|
+
* @param {number} pid The pid of the *spawned child* to demote.
|
|
70
|
+
*/
|
|
71
|
+
export function applyBackgroundPriority(pid) {
|
|
72
|
+
try {
|
|
73
|
+
if (!Number.isFinite(pid) || pid <= 0) return;
|
|
74
|
+
const pidStr = String(pid);
|
|
75
|
+
switch (process.platform) {
|
|
76
|
+
case 'darwin':
|
|
77
|
+
// taskpolicy -b -p PID → background band: CPU + IOPOL_THROTTLE + E-cores.
|
|
78
|
+
spawnBestEffort('taskpolicy', ['-b', '-p', pidStr]);
|
|
79
|
+
break;
|
|
80
|
+
case 'linux':
|
|
81
|
+
// ionice -c 3 → idle I/O class; chrt -b -p 0 → SCHED_BATCH (no root).
|
|
82
|
+
spawnBestEffort('ionice', ['-c', '3', '-p', pidStr]);
|
|
83
|
+
spawnBestEffort('chrt', ['-b', '-p', '0', pidStr]);
|
|
84
|
+
break;
|
|
85
|
+
default:
|
|
86
|
+
// Windows + others: no spawn-based lever; the native addon (A.5/A.6)
|
|
87
|
+
// covers Windows PROCESS_MODE_BACKGROUND_BEGIN in a later wave.
|
|
88
|
+
break;
|
|
89
|
+
}
|
|
90
|
+
} catch {
|
|
91
|
+
/* never throw — best-effort only */
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Memoized addon handle. `undefined` = not yet attempted; `null` = attempted
|
|
96
|
+
// and unavailable; otherwise the loaded module. Cached so repeated calls don't
|
|
97
|
+
// re-pay the require cost (or re-log a failure).
|
|
98
|
+
let _nativeAddon;
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Best-effort loader for the optional native background-priority N-API addon
|
|
102
|
+
* (`@sweet-search/bg-priority`, research §4.A A.5/A.6). Returns the addon
|
|
103
|
+
* module when the optionalDependency is installed and loads cleanly, else
|
|
104
|
+
* `null`. NEVER throws — a missing/failed native build is expected and the
|
|
105
|
+
* caller falls back to the spawn / `os.setPriority` levers.
|
|
106
|
+
*
|
|
107
|
+
* The module exposes `setBackgroundMode(enable: boolean): boolean` which demotes
|
|
108
|
+
* (or restores) the *calling* process/thread; it returns `true` when a native
|
|
109
|
+
* demotion was applied on this platform.
|
|
110
|
+
*
|
|
111
|
+
* @returns {null|{ setBackgroundMode: (enable: boolean) => boolean }} addon or null.
|
|
112
|
+
*/
|
|
113
|
+
export function loadNativePriorityAddon() {
|
|
114
|
+
if (_nativeAddon !== undefined) return _nativeAddon;
|
|
115
|
+
try {
|
|
116
|
+
// Use createRequire so this ESM module can load the CJS napi addon. The
|
|
117
|
+
// optionalDependency package name resolves to the per-platform prebuilt
|
|
118
|
+
// binary via the addon's own index.js loader.
|
|
119
|
+
const require = createRequire(import.meta.url);
|
|
120
|
+
const mod = require('@sweet-search/bg-priority');
|
|
121
|
+
_nativeAddon =
|
|
122
|
+
mod && typeof mod.setBackgroundMode === 'function' ? mod : null;
|
|
123
|
+
} catch {
|
|
124
|
+
// optionalDependency absent or failed to build → fall back. Expected.
|
|
125
|
+
_nativeAddon = null;
|
|
126
|
+
}
|
|
127
|
+
return _nativeAddon;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Self-demotion via the native addon: put the *current* process/thread into
|
|
132
|
+
* (`enable=true`) or out of (`enable=false`) OS background mode
|
|
133
|
+
* (Windows `PROCESS_MODE_BACKGROUND_BEGIN/END`, macOS per-thread
|
|
134
|
+
* `QOS_CLASS_BACKGROUND`/restore). The maintainer daemon calls this on ITSELF.
|
|
135
|
+
*
|
|
136
|
+
* Gated on `SWEET_SEARCH_MAINTAINER_NATIVE_PRIORITY==='1'` AND the addon being
|
|
137
|
+
* loaded. When either is false this is a no-op returning `false`, so the caller
|
|
138
|
+
* keeps its existing `os.setPriority(PRIORITY_LOW)` / spawn-based fallback
|
|
139
|
+
* (default behavior, unchanged). Best-effort, NEVER throws.
|
|
140
|
+
*
|
|
141
|
+
* Caveats (documented in the addon): on Windows, BACKGROUND_BEGIN empties the
|
|
142
|
+
* working set — only enable during bulk/idle phases and call `false` before
|
|
143
|
+
* serving latency-sensitive work. On macOS, QoS is per-thread and covers only
|
|
144
|
+
* threads that call it (ORT spins its own threads); NEVER mix raw
|
|
145
|
+
* `setpriority`/`taskpolicy` with QoS on the same thread.
|
|
146
|
+
*
|
|
147
|
+
* @param {boolean} enable true → enter background mode; false → restore.
|
|
148
|
+
* @returns {boolean} true if a native demotion/restoration was applied.
|
|
149
|
+
*/
|
|
150
|
+
export function setNativeBackgroundMode(enable) {
|
|
151
|
+
try {
|
|
152
|
+
if (process.env.SWEET_SEARCH_MAINTAINER_NATIVE_PRIORITY !== '1') return false;
|
|
153
|
+
const addon = loadNativePriorityAddon();
|
|
154
|
+
if (!addon) return false;
|
|
155
|
+
return addon.setBackgroundMode(Boolean(enable)) === true;
|
|
156
|
+
} catch {
|
|
157
|
+
/* never throw — best-effort only */
|
|
158
|
+
return false;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Global RSS-budget soft-eviction coordinator (research §4.D items D.3 + D.4).
|
|
3
|
+
*
|
|
4
|
+
* THE PROBLEM: N resident, model-loaded daemons (one search-server + one
|
|
5
|
+
* maintainer per repo) each hold the ORT session resident forever (#25325), so
|
|
6
|
+
* a dev who hops across ~8 repos accrues ~16 GB of resident daemons with no
|
|
7
|
+
* eviction/cap/TTL. D.1 idle-TTL (G4) collapses idle daemons; D.3 here adds a
|
|
8
|
+
* SAFETY CAP keyed on real system memory pressure, so even a fleet of *active*
|
|
9
|
+
* daemons cannot grow past a soft budget.
|
|
10
|
+
*
|
|
11
|
+
* WHAT IT IS (D.3): a coordinator — an unref'd ~30s timer running inside every
|
|
12
|
+
* registered daemon — that sums the RSS of all registered daemons and, when the
|
|
13
|
+
* total crosses `SWEET_SEARCH_RSS_BUDGET_FRACTION * os.totalmem()`, SIGTERMs the
|
|
14
|
+
* single longest-idle daemon (the one whose `lastActivityMs` is oldest). The
|
|
15
|
+
* evicted daemon exits cleanly (G4 SIGTERM handler) and respawns on its next use
|
|
16
|
+
* via the existing O_EXCL launch trigger, so the index is unchanged after
|
|
17
|
+
* respawn — this is a *footprint* policy, not a correctness one.
|
|
18
|
+
*
|
|
19
|
+
* WHAT IT IS NOT: it is **not** a V8 heap cap (`--max-old-space-size`) — that is
|
|
20
|
+
* explicitly forbidden on this hardware (`feedback_no_memory_cap`). The budget
|
|
21
|
+
* is a soft, system-RAM-scaled eviction threshold: 0.60 → ~9.6 GB on a 16 GB
|
|
22
|
+
* laptop, ~76 GB on a 128 GB workstation. No per-machine config; it auto-scales.
|
|
23
|
+
*
|
|
24
|
+
* WHAT IT EXTENDS, NOT DUPLICATES: the longest-idle SELECTION reuses
|
|
25
|
+
* `selectEvictionTargets` from the search daemon-registry (the same convergence-
|
|
26
|
+
* safe "only shed peers less-recently-active than self" rule that backs
|
|
27
|
+
* `SWEET_SEARCH_MAX_DAEMONS`). The LRU *count*-cap (search-server.js) and this
|
|
28
|
+
* RSS *budget*-cap are two thresholds over the same registry-style structure;
|
|
29
|
+
* this module owns only the RSS dimension.
|
|
30
|
+
*
|
|
31
|
+
* D.4 memory-pressure: on Linux we additionally read PSI
|
|
32
|
+
* `/proc/pressure/memory` (`some avg10`); a non-zero stall is an earlier,
|
|
33
|
+
* kernel-supplied "drop idle daemons before the OS OOM-kills" signal than RSS
|
|
34
|
+
* alone. On macOS the equivalent (`DISPATCH_SOURCE_TYPE_MEMORYPRESSURE`) has no
|
|
35
|
+
* confirmed Node binding and needs a C addon — that is a DEFERRED openDecision
|
|
36
|
+
* (research §6); macOS falls back to the pure-JS RSS poller below.
|
|
37
|
+
*
|
|
38
|
+
* DESIGN CONTRACT:
|
|
39
|
+
* - Tier-aware default. With `SWEET_SEARCH_RSS_BUDGET_FRACTION` unset the
|
|
40
|
+
* fraction comes from the system-RAM tier (`resolveMaintainerMemoryProfile`):
|
|
41
|
+
* OFF on roomy hosts (>24 GiB → NO registry write, NO timer, NO eviction —
|
|
42
|
+
* byte-identical to before), a soft cap on small-RAM hosts (the OOM case).
|
|
43
|
+
* An explicit env value always overrides: a parseable fraction in (0,1]
|
|
44
|
+
* enables; '', '0', or garbage disables the coordinator entirely.
|
|
45
|
+
* - Best-effort and NEVER throws. Every RSS read, PSI read, registry write,
|
|
46
|
+
* and signal is wrapped; a failure degrades to "do nothing this tick".
|
|
47
|
+
* - The timer is unref'd: it never keeps a daemon's event loop alive.
|
|
48
|
+
* - Soft eviction only: it SIGTERMs (graceful) — never SIGKILL, never self.
|
|
49
|
+
*/
|
|
50
|
+
|
|
51
|
+
import fs from 'node:fs/promises';
|
|
52
|
+
import os from 'node:os';
|
|
53
|
+
import path from 'node:path';
|
|
54
|
+
import { spawnSync } from 'node:child_process';
|
|
55
|
+
|
|
56
|
+
import { selectEvictionTargets } from '../search/daemon-registry.js';
|
|
57
|
+
import { resolveMaintainerMemoryProfile } from '../incremental-indexing/domain/interval-autotune.mjs';
|
|
58
|
+
|
|
59
|
+
const DEFAULT_REGISTRY_FILE = 'sweet-search-rss-daemons.json';
|
|
60
|
+
const POLL_INTERVAL_MS = 30_000;
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Path to the shared RSS registry file. Distinct from the search daemon-registry
|
|
64
|
+
* file (which is the LRU-count surface): this one also tracks maintainers and is
|
|
65
|
+
* RAM-keyed. Override via SWEET_SEARCH_RSS_REGISTRY for tests; otherwise a single
|
|
66
|
+
* shared file under the OS tmp dir so every daemon on the host coordinates.
|
|
67
|
+
*/
|
|
68
|
+
export function rssRegistryPath(env = process.env) {
|
|
69
|
+
return env.SWEET_SEARCH_RSS_REGISTRY || path.join(os.tmpdir(), DEFAULT_REGISTRY_FILE);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Parse + validate the budget fraction gate. Returns a number in (0,1] when the
|
|
74
|
+
* coordinator is enabled, or `null` (disabled) for unset/empty/zero/garbage.
|
|
75
|
+
* This is the single source of the default-OFF contract.
|
|
76
|
+
*/
|
|
77
|
+
export function budgetFraction(env = process.env, totalMemBytes = os.totalmem()) {
|
|
78
|
+
const raw = env.SWEET_SEARCH_RSS_BUDGET_FRACTION;
|
|
79
|
+
if (raw != null) {
|
|
80
|
+
// Explicitly set (incl. '' / '0' / garbage → disabled): env wins over tier.
|
|
81
|
+
const f = Number(raw);
|
|
82
|
+
return (Number.isFinite(f) && f > 0 && f <= 1) ? f : null;
|
|
83
|
+
}
|
|
84
|
+
// Unset → auto from the system-RAM tier: small-RAM hosts get a soft cap, roomy
|
|
85
|
+
// hosts get none (null). No per-machine config; it auto-scales with RAM.
|
|
86
|
+
return resolveMaintainerMemoryProfile({ totalMemBytes }).rssBudgetFraction;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Whether the coordinator is enabled (explicit fraction OR a RAM-tier default). */
|
|
90
|
+
export function isEnabled(env = process.env, totalMemBytes = os.totalmem()) {
|
|
91
|
+
return budgetFraction(env, totalMemBytes) !== null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* The soft budget in bytes: `fraction * totalmem`. Auto-scales with system RAM
|
|
96
|
+
* so there is no per-machine config (4 GB-ish on a 16 GB box at 0.6, ~76 GB on
|
|
97
|
+
* 128 GB). Returns 0 when disabled.
|
|
98
|
+
*/
|
|
99
|
+
export function budgetBytes(env = process.env, totalMem = os.totalmem()) {
|
|
100
|
+
const f = budgetFraction(env, totalMem);
|
|
101
|
+
if (f === null) return 0;
|
|
102
|
+
return Math.floor(f * totalMem);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Total RSS over budget? Pure predicate (testable without any I/O). */
|
|
106
|
+
export function isOverBudget(totalRssBytes, budgetBytesValue) {
|
|
107
|
+
return (
|
|
108
|
+
Number.isFinite(totalRssBytes) &&
|
|
109
|
+
Number.isFinite(budgetBytesValue) &&
|
|
110
|
+
budgetBytesValue > 0 &&
|
|
111
|
+
totalRssBytes > budgetBytesValue
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Read the resident-set size (bytes) of an arbitrary pid, best-effort and
|
|
117
|
+
* cross-platform. Returns a non-negative integer, or 0 when unknown (dead pid,
|
|
118
|
+
* unsupported platform, permission denied) — an unknown daemon contributes 0 to
|
|
119
|
+
* the sum, so a transient read failure can only UNDER-count (never spuriously
|
|
120
|
+
* evict). For the calling process we prefer the cheap, exact in-process value.
|
|
121
|
+
*
|
|
122
|
+
* - self: `process.memoryUsage().rss` (no spawn).
|
|
123
|
+
* - Linux: `/proc/<pid>/statm` field 2 (resident pages) × page size.
|
|
124
|
+
* - macOS / BSD: `ps -o rss= -p <pid>` (KiB) × 1024.
|
|
125
|
+
* - Windows / others: 0 (no cheap spawn-free reader; native addon later).
|
|
126
|
+
*
|
|
127
|
+
* @param {number} pid
|
|
128
|
+
* @returns {Promise<number>} RSS in bytes (0 when unknown).
|
|
129
|
+
*/
|
|
130
|
+
export async function readProcessRss(pid) {
|
|
131
|
+
try {
|
|
132
|
+
const n = Number(pid);
|
|
133
|
+
if (!Number.isInteger(n) || n <= 0) return 0;
|
|
134
|
+
if (n === process.pid) {
|
|
135
|
+
try { return process.memoryUsage().rss || 0; } catch { return 0; }
|
|
136
|
+
}
|
|
137
|
+
if (process.platform === 'linux') {
|
|
138
|
+
try {
|
|
139
|
+
const raw = await fs.readFile(`/proc/${n}/statm`, 'utf-8');
|
|
140
|
+
const resPages = Number(raw.trim().split(/\s+/)[1]);
|
|
141
|
+
if (!Number.isFinite(resPages) || resPages <= 0) return 0;
|
|
142
|
+
const pageSize = typeof os.constants?.UV_PAGESIZE === 'number'
|
|
143
|
+
? os.constants.UV_PAGESIZE
|
|
144
|
+
: 4096;
|
|
145
|
+
return resPages * pageSize;
|
|
146
|
+
} catch {
|
|
147
|
+
return 0;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
if (process.platform === 'darwin') {
|
|
151
|
+
// `ps` RSS is in KiB. spawnSync is fine here: coarse 30s cadence, tiny output.
|
|
152
|
+
const r = spawnSync('ps', ['-o', 'rss=', '-p', String(n)], { encoding: 'utf-8', timeout: 2000 });
|
|
153
|
+
if (r.status !== 0 || !r.stdout) return 0;
|
|
154
|
+
const kib = Number(r.stdout.trim());
|
|
155
|
+
return Number.isFinite(kib) && kib > 0 ? kib * 1024 : 0;
|
|
156
|
+
}
|
|
157
|
+
return 0; // Windows + others: no spawn-free reader yet (deferred native addon).
|
|
158
|
+
} catch {
|
|
159
|
+
return 0;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* D.4 Linux memory-pressure reader (PSI). Returns the `some avg10` stall
|
|
165
|
+
* percentage from `/proc/pressure/memory` (0–100), or `null` when unavailable
|
|
166
|
+
* (non-Linux, kernel without PSI, read error). A non-zero `some avg10` means
|
|
167
|
+
* tasks were stalled on memory in the last 10s — an earlier OOM-warning than
|
|
168
|
+
* raw RSS, so the coordinator treats any positive value as "prefer to evict".
|
|
169
|
+
*
|
|
170
|
+
* macOS note: the kernel equivalent is `DISPATCH_SOURCE_TYPE_MEMORYPRESSURE`,
|
|
171
|
+
* which has no confirmed Node binding and needs a C addon (DEFERRED
|
|
172
|
+
* openDecision, research §6). On macOS this returns null and the coordinator
|
|
173
|
+
* relies on the RSS poller alone.
|
|
174
|
+
*
|
|
175
|
+
* @returns {Promise<number|null>} `some avg10` (0–100) or null.
|
|
176
|
+
*/
|
|
177
|
+
export async function readLinuxMemoryPressure() {
|
|
178
|
+
try {
|
|
179
|
+
if (process.platform !== 'linux') return null;
|
|
180
|
+
const raw = await fs.readFile('/proc/pressure/memory', 'utf-8');
|
|
181
|
+
// Format: "some avg10=0.00 avg60=0.00 avg300=0.00 total=0\nfull ..."
|
|
182
|
+
for (const line of raw.split('\n')) {
|
|
183
|
+
if (!line.startsWith('some ')) continue;
|
|
184
|
+
const m = line.match(/avg10=([\d.]+)/);
|
|
185
|
+
if (m) {
|
|
186
|
+
const v = Number(m[1]);
|
|
187
|
+
return Number.isFinite(v) ? v : null;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
return null;
|
|
191
|
+
} catch {
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// --- shared-registry I/O (best-effort, atomic; mirrors daemon-registry.js) ---
|
|
197
|
+
|
|
198
|
+
async function readRssRegistry(env = process.env) {
|
|
199
|
+
try {
|
|
200
|
+
const raw = await fs.readFile(rssRegistryPath(env), 'utf-8');
|
|
201
|
+
const parsed = JSON.parse(raw);
|
|
202
|
+
const daemons = parsed && typeof parsed === 'object' ? parsed.daemons : null;
|
|
203
|
+
return daemons && typeof daemons === 'object' ? daemons : {};
|
|
204
|
+
} catch {
|
|
205
|
+
return {};
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
async function writeRssRegistryAtomic(daemons, env = process.env) {
|
|
210
|
+
const target = rssRegistryPath(env);
|
|
211
|
+
const tmp = `${target}.${process.pid}.tmp`;
|
|
212
|
+
try {
|
|
213
|
+
await fs.writeFile(tmp, JSON.stringify({ daemons }), { mode: 0o600 });
|
|
214
|
+
await fs.rename(tmp, target);
|
|
215
|
+
return true;
|
|
216
|
+
} catch {
|
|
217
|
+
try { await fs.unlink(tmp); } catch { /* ignore */ }
|
|
218
|
+
return false;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** Is a pid alive right now? `kill -0`; EPERM (other-user) counts as alive. */
|
|
223
|
+
function pidAlive(pid) {
|
|
224
|
+
const n = Number(pid);
|
|
225
|
+
if (!Number.isInteger(n) || n <= 0) return false;
|
|
226
|
+
try {
|
|
227
|
+
process.kill(n, 0);
|
|
228
|
+
return true;
|
|
229
|
+
} catch (err) {
|
|
230
|
+
return !!(err && err.code === 'EPERM');
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* Drop registry entries whose process is gone (the only liveness gate this
|
|
236
|
+
* coordinator needs — unlike the search registry it does not socket-probe, since
|
|
237
|
+
* maintainers have no health socket), persist if anything changed, and return
|
|
238
|
+
* the surviving entries.
|
|
239
|
+
*/
|
|
240
|
+
async function pruneRegistry(env = process.env, alive = pidAlive) {
|
|
241
|
+
const daemons = await readRssRegistry(env);
|
|
242
|
+
const live = [];
|
|
243
|
+
const liveMap = {};
|
|
244
|
+
for (const [key, entry] of Object.entries(daemons)) {
|
|
245
|
+
if (!entry || typeof entry !== 'object') continue;
|
|
246
|
+
if (alive(entry.pid)) {
|
|
247
|
+
live.push(entry);
|
|
248
|
+
liveMap[key] = entry;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (Object.keys(liveMap).length !== Object.keys(daemons).length) {
|
|
252
|
+
await writeRssRegistryAtomic(liveMap, env);
|
|
253
|
+
}
|
|
254
|
+
return live;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* One coordinator tick (exported for tests; takes injectable readers so the
|
|
259
|
+
* eviction decision is testable with zero real processes/signals/files):
|
|
260
|
+
*
|
|
261
|
+
* 1. prune dead entries, list the live ones;
|
|
262
|
+
* 2. read each daemon's RSS, sum it, and stamp each entry with its rss;
|
|
263
|
+
* 3. read Linux PSI (if any) — a positive `some avg10` is an early pressure
|
|
264
|
+
* signal that forces an eviction pass even slightly under the RSS budget;
|
|
265
|
+
* 4. if over budget (or under pressure with >1 daemon), pick the single
|
|
266
|
+
* longest-idle peer via `selectEvictionTargets` (self-safe convergence)
|
|
267
|
+
* and SIGTERM it.
|
|
268
|
+
*
|
|
269
|
+
* Returns a small diagnostic record. Best-effort: never throws.
|
|
270
|
+
*
|
|
271
|
+
* @param {object} deps
|
|
272
|
+
* @param {object} [deps.env]
|
|
273
|
+
* @param {number} [deps.selfPid]
|
|
274
|
+
* @param {(pid:number)=>Promise<number>} [deps.rssReader]
|
|
275
|
+
* @param {()=>Promise<number|null>} [deps.pressureReader]
|
|
276
|
+
* @param {(pid:number)=>void} [deps.signal] Eviction action (default SIGTERM).
|
|
277
|
+
* @param {number} [deps.totalMem] System RAM (default os.totalmem()); injectable
|
|
278
|
+
* so the budget is deterministic in tests regardless of the host's real RAM.
|
|
279
|
+
* @param {(pid:number)=>boolean} [deps.aliveProbe] Liveness probe (default
|
|
280
|
+
* `kill -0`); injectable so tests can mark seeded fake pids as live.
|
|
281
|
+
*/
|
|
282
|
+
export async function runEvictionTick({
|
|
283
|
+
env = process.env,
|
|
284
|
+
selfPid = process.pid,
|
|
285
|
+
rssReader = readProcessRss,
|
|
286
|
+
pressureReader = readLinuxMemoryPressure,
|
|
287
|
+
signal = (pid) => { try { process.kill(pid, 'SIGTERM'); } catch { /* best-effort */ } },
|
|
288
|
+
totalMem = os.totalmem(),
|
|
289
|
+
aliveProbe = pidAlive,
|
|
290
|
+
} = {}) {
|
|
291
|
+
const result = { enabled: false, totalRss: 0, budget: 0, over: false, pressure: null, evicted: null, liveCount: 0 };
|
|
292
|
+
try {
|
|
293
|
+
if (!isEnabled(env)) return result;
|
|
294
|
+
result.enabled = true;
|
|
295
|
+
result.budget = budgetBytes(env, totalMem);
|
|
296
|
+
|
|
297
|
+
const live = await pruneRegistry(env, aliveProbe);
|
|
298
|
+
result.liveCount = live.length;
|
|
299
|
+
|
|
300
|
+
// Sum RSS, stamping each entry so selectEvictionTargets sees fresh data.
|
|
301
|
+
let total = 0;
|
|
302
|
+
for (const entry of live) {
|
|
303
|
+
const rss = await rssReader(entry.pid);
|
|
304
|
+
entry.rss = rss;
|
|
305
|
+
total += rss;
|
|
306
|
+
}
|
|
307
|
+
result.totalRss = total;
|
|
308
|
+
result.pressure = await pressureReader();
|
|
309
|
+
|
|
310
|
+
const over = isOverBudget(total, result.budget);
|
|
311
|
+
// PSI: any positive `some avg10` stall is an early OOM-warning. Only act on
|
|
312
|
+
// it when more than one daemon is resident (evicting the sole daemon would
|
|
313
|
+
// just trigger a respawn for no footprint relief) AND we are at least
|
|
314
|
+
// approaching the budget (≥80%), so a momentary system-wide stall unrelated
|
|
315
|
+
// to our daemons cannot churn the fleet.
|
|
316
|
+
const underPressure =
|
|
317
|
+
Number.isFinite(result.pressure) && result.pressure > 0 &&
|
|
318
|
+
live.length > 1 && result.budget > 0 && total >= 0.8 * result.budget;
|
|
319
|
+
result.over = over;
|
|
320
|
+
|
|
321
|
+
if (!over && !underPressure) return result;
|
|
322
|
+
|
|
323
|
+
// Pick the single longest-idle peer. selectEvictionTargets sorts oldest
|
|
324
|
+
// lastActivityMs first and only returns peers strictly older than self, so
|
|
325
|
+
// every daemon running this tick converges on the same victim set without
|
|
326
|
+
// over-evicting (identical convergence proof to the LRU count-cap).
|
|
327
|
+
const targets = selectEvictionTargets(live, selfPid, 1);
|
|
328
|
+
if (targets.length === 0) return result;
|
|
329
|
+
const victim = targets[0];
|
|
330
|
+
signal(victim.pid);
|
|
331
|
+
result.evicted = victim.pid;
|
|
332
|
+
return result;
|
|
333
|
+
} catch {
|
|
334
|
+
return result; // never throw — best-effort coordinator
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
// --- public registration API (the seam G4 wired in index-maintainer.mjs) ---
|
|
339
|
+
|
|
340
|
+
/** Module-level coordinator handle so multiple registrations share ONE timer. */
|
|
341
|
+
let coordinatorTimer = null;
|
|
342
|
+
let registeredCount = 0;
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* Register THIS daemon with the RSS-budget coordinator. Matches the seam G4
|
|
346
|
+
* wired into index-maintainer.mjs:
|
|
347
|
+
*
|
|
348
|
+
* const mod = await import('./rss-budget.mjs');
|
|
349
|
+
* rssRegistration = await mod.registerDaemon({ pid, stateDir, kind });
|
|
350
|
+
* // ... later, on shutdown:
|
|
351
|
+
* await rssRegistration.unregister();
|
|
352
|
+
*
|
|
353
|
+
* Upserts an entry into the shared RSS registry and, on the first registration
|
|
354
|
+
* in this process, starts the unref'd ~30s coordinator timer. Returns an
|
|
355
|
+
* `{ unregister }` handle (and `{ touch }` so a daemon can refresh its real
|
|
356
|
+
* activity timestamp — used by search-server-style query routes; the maintainer
|
|
357
|
+
* leaves lastActivityMs at registration time, which is correct: an idle
|
|
358
|
+
* maintainer is exactly what we want to evict first).
|
|
359
|
+
*
|
|
360
|
+
* Best-effort: with the gate off this is a no-op returning a harmless handle, so
|
|
361
|
+
* the caller's teardown (`rssRegistration?.unregister?.()`) is always safe.
|
|
362
|
+
*
|
|
363
|
+
* @param {object} opts
|
|
364
|
+
* @param {number} opts.pid
|
|
365
|
+
* @param {string} [opts.stateDir]
|
|
366
|
+
* @param {string} [opts.kind] 'maintainer' | 'search' | etc. (diagnostic only)
|
|
367
|
+
* @param {object} [opts.env]
|
|
368
|
+
* @returns {Promise<{ unregister: () => Promise<void>, touch: (ms?: number) => Promise<void> }>}
|
|
369
|
+
*/
|
|
370
|
+
export async function registerDaemon({ pid = process.pid, stateDir = null, kind = 'unknown', env = process.env } = {}) {
|
|
371
|
+
const noop = { unregister: async () => {}, touch: async () => {} };
|
|
372
|
+
try {
|
|
373
|
+
if (!isEnabled(env)) return noop;
|
|
374
|
+
|
|
375
|
+
const now = Date.now();
|
|
376
|
+
const entry = { pid, stateDir, kind, startedAt: now, lastActivityMs: now, rss: 0 };
|
|
377
|
+
try {
|
|
378
|
+
const daemons = await readRssRegistry(env);
|
|
379
|
+
daemons[String(pid)] = entry;
|
|
380
|
+
await writeRssRegistryAtomic(daemons, env);
|
|
381
|
+
} catch { /* best-effort: a failed write just means we aren't counted */ }
|
|
382
|
+
|
|
383
|
+
registeredCount += 1;
|
|
384
|
+
if (!coordinatorTimer) {
|
|
385
|
+
coordinatorTimer = setInterval(() => {
|
|
386
|
+
runEvictionTick({ env, selfPid: pid }).catch(() => {});
|
|
387
|
+
}, POLL_INTERVAL_MS);
|
|
388
|
+
if (coordinatorTimer.unref) coordinatorTimer.unref();
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
const unregister = async () => {
|
|
392
|
+
try {
|
|
393
|
+
const daemons = await readRssRegistry(env);
|
|
394
|
+
if (String(pid) in daemons) {
|
|
395
|
+
delete daemons[String(pid)];
|
|
396
|
+
await writeRssRegistryAtomic(daemons, env);
|
|
397
|
+
}
|
|
398
|
+
} catch { /* best-effort */ }
|
|
399
|
+
registeredCount = Math.max(0, registeredCount - 1);
|
|
400
|
+
if (registeredCount === 0 && coordinatorTimer) {
|
|
401
|
+
clearInterval(coordinatorTimer);
|
|
402
|
+
coordinatorTimer = null;
|
|
403
|
+
}
|
|
404
|
+
};
|
|
405
|
+
|
|
406
|
+
const touch = async (ms = Date.now()) => {
|
|
407
|
+
try {
|
|
408
|
+
const daemons = await readRssRegistry(env);
|
|
409
|
+
if (daemons[String(pid)]) {
|
|
410
|
+
daemons[String(pid)].lastActivityMs = ms;
|
|
411
|
+
await writeRssRegistryAtomic(daemons, env);
|
|
412
|
+
}
|
|
413
|
+
} catch { /* best-effort */ }
|
|
414
|
+
};
|
|
415
|
+
|
|
416
|
+
return { unregister, touch };
|
|
417
|
+
} catch {
|
|
418
|
+
return noop; // never throw — a registration failure must not break startup
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/** Test-only: read the live RSS registry map. */
|
|
423
|
+
export async function _readRegistryForTest(env = process.env) {
|
|
424
|
+
return readRssRegistry(env);
|
|
425
|
+
}
|