switchroom 0.20.12 → 0.20.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +1 -0
- package/dist/auth-broker/index.js +5 -1
- package/dist/cli/notion-write-pretool.mjs +1 -0
- package/dist/cli/self-improve-stop.mjs +13 -1
- package/dist/cli/switchroom.js +1472 -89
- package/dist/host-control/main.js +6 -2
- package/dist/vault/approvals/kernel-server.js +5 -1
- package/dist/vault/broker/server.js +5 -1
- package/package.json +7 -4
- package/skills/switchroom-release/SKILL.md +3 -2
- package/telegram-plugin/bunfig.toml +9 -5
- package/telegram-plugin/dist/gateway/gateway.js +169 -68
- package/telegram-plugin/gateway/captured-answer-resume.ts +23 -1
- package/telegram-plugin/gateway/gateway.ts +1 -0
- package/telegram-plugin/gateway/outbound-send-path.ts +111 -6
- package/telegram-plugin/gateway/outbox-sweep.ts +69 -0
- package/telegram-plugin/gateway/stream-render.ts +4 -0
- package/telegram-plugin/hooks/audience-classify.d.mts +26 -0
- package/telegram-plugin/hooks/audience-classify.mjs +193 -0
- package/telegram-plugin/hooks/silent-end-interrupt-stop.mjs +31 -1
- package/telegram-plugin/hooks/silent-end-scan.mjs +8 -1
- package/telegram-plugin/outbox.ts +69 -3
- package/telegram-plugin/silent-end.ts +48 -5
- package/telegram-plugin/tests/captured-answer-resume.test.ts +26 -11
- package/telegram-plugin/tests/hindsight-bank-preload.test.ts +50 -0
- package/telegram-plugin/tests/outbox-live-path-review-4490.test.ts +613 -0
- package/telegram-plugin/tests/outbox-self-improve-review.test.ts +401 -0
package/dist/cli/switchroom.js
CHANGED
|
@@ -2120,7 +2120,7 @@ var init_esm = __esm(() => {
|
|
|
2120
2120
|
});
|
|
2121
2121
|
|
|
2122
2122
|
// src/build-info.ts
|
|
2123
|
-
var VERSION = "0.20.
|
|
2123
|
+
var VERSION = "0.20.13", COMMIT_SHA = "a73b0fd2";
|
|
2124
2124
|
|
|
2125
2125
|
// src/cli/resolve-version.ts
|
|
2126
2126
|
import { existsSync, readFileSync } from "node:fs";
|
|
@@ -14034,6 +14034,7 @@ var init_schema = __esm(() => {
|
|
|
14034
14034
|
}).describe("Per-operation LLM override. Every field optional; an unset field (or " + "an omitted op block) inherits the global `hindsight.llm.*`, which is " + "already the engine's fallback \u2014 switchroom emits only the vars set.");
|
|
14035
14035
|
HindsightConfigSchema = exports_external.object({
|
|
14036
14036
|
gpu: exports_external.boolean().optional().describe("Force GPU passthrough for the hindsight container on (`true`) or off " + "(`false`), overriding host autodetection in BOTH directions. Absent " + "(the default) \u2192 autodetect from the persisted host-capabilities " + "verdict (`~/.switchroom/host-capabilities.json`), which enables " + "`--gpus all` only when that file proves BOTH a GPU and the nvidia " + "container toolkit. Set `true` when that verdict is wrong or unreadable " + "and you know the host has a working toolkit \u2014 switchroom cannot verify " + "it for you, and `docker run --gpus all` hard-fails container create on " + "a host without one. Set `false` to pin the container to CPU on a GPU " + "host. This is also the declarative opt-out for the recreate-time GPU " + "drop guard (`switchroom memory setup --recreate` refuses to silently " + "turn a GPU container into a CPU one). `--gpu`/`--no-gpu` on `memory " + "setup` override this for a single run."),
|
|
14037
|
+
mem_limit: exports_external.string().regex(/^\d+(\.\d+)?\s*[bkmgt]?b?$/i, "hindsight.mem_limit must be a docker memory size, e.g. `24g` or `16384m`").optional().describe("Hard memory cap for the hindsight container \u2014 docker `--memory` on the " + "`memory setup` path and `mem_limit:` in the emitted compose snippet. A " + "docker size string (`24g`, `16384m`). Absent \u2192 switchroom's default " + "(`HINDSIGHT_DEFAULT_MEM_LIMIT`, 16g). This is a docker HostConfig flag, " + "not a container env var, which is why it is a first-class field here " + "rather than an `env:` key. **Move it together with " + "`env.SWITCHROOM_HINDSIGHT_PG_SHARED_BUFFERS`**: Postgres pins the " + "buffer pool as unreclaimable shared memory inside this cap, so a cap " + "that does not clear `shared_buffers` by the container's app working " + "set plus its page-cache floor puts Postgres in a near-OOM " + "configuration. switchroom warns, naming both numbers, when it does " + "not. A value below the container's memory RESERVATION (4g) is " + "rejected outright \u2014 docker will not create such a container."),
|
|
14037
14038
|
cp_access_key: exports_external.string().min(1).optional().describe("Access key for the hindsight control-plane DASHBOARD (upstream " + "`HINDSIGHT_CP_ACCESS_KEY`, port 9999). Literal or \u2014 strongly preferred " + "\u2014 a `vault:` reference such as `vault:hindsight_cp_access_key`, read " + "through the broker at container-launch time. This is the ONLY thing " + "that arms the dashboard's login: upstream's middleware short-circuits " + "to `next()` when the var is unset, so an unset key means the dashboard " + "has no authentication at all, not weak authentication. Because of that, " + "leaving it unset is FAIL-CLOSED: switchroom pins " + "`HINDSIGHT_CP_HOSTNAME=127.0.0.1` so the loginless dashboard is " + "reachable only from the host, and warns. Set this to serve the " + "dashboard on the LAN/tailnet."),
|
|
14038
14039
|
llm: exports_external.object({
|
|
14039
14040
|
provider: exports_external.string().min(1).optional().describe("Hindsight LLM provider (upstream `HINDSIGHT_API_LLM_PROVIDER`). " + "Defaults to `claude-code` (subscription-honest, broker-fed OAuth). " + "Any litellm-routable provider the upstream image supports is valid. " + "Serves as the GLOBAL default for every op absent a per-op override."),
|
|
@@ -16005,6 +16006,66 @@ var init_host_capabilities = __esm(() => {
|
|
|
16005
16006
|
function pgMib(mib) {
|
|
16006
16007
|
return `${mib}MB`;
|
|
16007
16008
|
}
|
|
16009
|
+
function parseDockerSizeToMib(size) {
|
|
16010
|
+
const m = size.trim().match(/^(\d+(?:\.\d+)?)\s*([bkmgt])?b?$/i);
|
|
16011
|
+
if (!m)
|
|
16012
|
+
return null;
|
|
16013
|
+
const n = Number(m[1]);
|
|
16014
|
+
if (!Number.isFinite(n))
|
|
16015
|
+
return null;
|
|
16016
|
+
switch ((m[2] ?? "b").toLowerCase()) {
|
|
16017
|
+
case "b":
|
|
16018
|
+
return n / (1024 * 1024);
|
|
16019
|
+
case "k":
|
|
16020
|
+
return n / 1024;
|
|
16021
|
+
case "m":
|
|
16022
|
+
return n;
|
|
16023
|
+
case "g":
|
|
16024
|
+
return n * 1024;
|
|
16025
|
+
case "t":
|
|
16026
|
+
return n * 1024 * 1024;
|
|
16027
|
+
default:
|
|
16028
|
+
return null;
|
|
16029
|
+
}
|
|
16030
|
+
}
|
|
16031
|
+
function parsePgSizeToMib(value) {
|
|
16032
|
+
const raw = value.trim();
|
|
16033
|
+
if (raw.toLowerCase() === "off")
|
|
16034
|
+
return HINDSIGHT_PG0_FALLBACK_SHARED_BUFFERS_MIB;
|
|
16035
|
+
const m = raw.match(/^(\d+(?:\.\d+)?)\s*(kB|MB|GB|TB)?$/i);
|
|
16036
|
+
if (!m)
|
|
16037
|
+
return null;
|
|
16038
|
+
const n = Number(m[1]);
|
|
16039
|
+
if (!Number.isFinite(n))
|
|
16040
|
+
return null;
|
|
16041
|
+
switch ((m[2] ?? "").toUpperCase()) {
|
|
16042
|
+
case "":
|
|
16043
|
+
return n * 8 / 1024;
|
|
16044
|
+
case "KB":
|
|
16045
|
+
return n / 1024;
|
|
16046
|
+
case "MB":
|
|
16047
|
+
return n;
|
|
16048
|
+
case "GB":
|
|
16049
|
+
return n * 1024;
|
|
16050
|
+
case "TB":
|
|
16051
|
+
return n * 1024 * 1024;
|
|
16052
|
+
default:
|
|
16053
|
+
return null;
|
|
16054
|
+
}
|
|
16055
|
+
}
|
|
16056
|
+
function mib(n) {
|
|
16057
|
+
return n >= 1024 && n % 1024 === 0 ? `${n / 1024} GiB` : `${Math.round(n)} MiB`;
|
|
16058
|
+
}
|
|
16059
|
+
function hindsightMemBudgetWarning(input) {
|
|
16060
|
+
const capMib = parseDockerSizeToMib(input.memLimit);
|
|
16061
|
+
const bufMib = parsePgSizeToMib(input.sharedBuffers);
|
|
16062
|
+
if (capMib === null || bufMib === null)
|
|
16063
|
+
return null;
|
|
16064
|
+
const headroom = capMib - bufMib;
|
|
16065
|
+
if (headroom >= HINDSIGHT_PG_MIN_NON_BUFFER_MIB)
|
|
16066
|
+
return null;
|
|
16067
|
+
return `hindsight: container memory cap ${input.memLimit} (${mib(capMib)}) leaves only ` + `${mib(Math.max(headroom, 0))} above shared_buffers ${input.sharedBuffers} ` + `(${mib(bufMib)}) \u2014 Postgres pins that buffer pool as unreclaimable shared ` + `memory, so the container needs at least ${mib(HINDSIGHT_PG_MIN_NON_BUFFER_MIB)} ` + `beyond it (${mib(HINDSIGHT_PG_APP_ANON_MIB)} app working set + ` + `${mib(HINDSIGHT_PG_PAGE_CACHE_FLOOR_MIB)} page-cache floor). Raise ` + `\`hindsight.mem_limit\` to at least ` + `${mib(Math.ceil((bufMib + HINDSIGHT_PG_MIN_NON_BUFFER_MIB) / 1024) * 1024)}, or lower ` + "`hindsight.env.SWITCHROOM_HINDSIGHT_PG_SHARED_BUFFERS`.";
|
|
16068
|
+
}
|
|
16008
16069
|
function resolveHindsightPgOverrides(configEnv, processEnv = process.env) {
|
|
16009
16070
|
const out = new Map;
|
|
16010
16071
|
for (const key of HINDSIGHT_PG_ENV_KEYS) {
|
|
@@ -16034,10 +16095,11 @@ function hindsightPgEnv(overrides = new Map) {
|
|
|
16034
16095
|
}
|
|
16035
16096
|
return out;
|
|
16036
16097
|
}
|
|
16037
|
-
var HINDSIGHT_PG_MEM_LIMIT_MIB_FOR_DERIVATION, HINDSIGHT_PG_APP_ANON_MIB = 2560, HINDSIGHT_PG_PAGE_CACHE_FLOOR_MIB = 2048, HINDSIGHT_PG_SHARED_BUFFERS_BUDGET_MIB, HINDSIGHT_PG_DEFAULT_SHARED_BUFFERS_MIB = 6144, HINDSIGHT_PG_DEFAULT_EFFECTIVE_CACHE_SIZE_MIB = 12288, HINDSIGHT_PG_EFFECTIVE_CACHE_SIZE_ENV = "SWITCHROOM_HINDSIGHT_PG_EFFECTIVE_CACHE_SIZE", HINDSIGHT_PG_SHARED_BUFFERS_ENV = "SWITCHROOM_HINDSIGHT_PG_SHARED_BUFFERS", HINDSIGHT_PG_FSYNC_ENV = "SWITCHROOM_HINDSIGHT_PG_FSYNC", HINDSIGHT_PG_DEFAULT_FSYNC = "on", HINDSIGHT_PG_DEFAULTS, HINDSIGHT_PG_ENV_KEYS;
|
|
16098
|
+
var HINDSIGHT_PG_MEM_LIMIT_MIB_FOR_DERIVATION, HINDSIGHT_PG_APP_ANON_MIB = 2560, HINDSIGHT_PG_PAGE_CACHE_FLOOR_MIB = 2048, HINDSIGHT_PG_SHARED_BUFFERS_BUDGET_MIB, HINDSIGHT_PG_DEFAULT_SHARED_BUFFERS_MIB = 6144, HINDSIGHT_PG_DEFAULT_EFFECTIVE_CACHE_SIZE_MIB = 12288, HINDSIGHT_PG0_FALLBACK_SHARED_BUFFERS_MIB = 256, HINDSIGHT_PG_MIN_NON_BUFFER_MIB, HINDSIGHT_PG_EFFECTIVE_CACHE_SIZE_ENV = "SWITCHROOM_HINDSIGHT_PG_EFFECTIVE_CACHE_SIZE", HINDSIGHT_PG_SHARED_BUFFERS_ENV = "SWITCHROOM_HINDSIGHT_PG_SHARED_BUFFERS", HINDSIGHT_PG_FSYNC_ENV = "SWITCHROOM_HINDSIGHT_PG_FSYNC", HINDSIGHT_PG_DEFAULT_FSYNC = "on", HINDSIGHT_PG_DEFAULTS, HINDSIGHT_PG_ENV_KEYS;
|
|
16038
16099
|
var init_hindsight_pg_defaults = __esm(() => {
|
|
16039
16100
|
HINDSIGHT_PG_MEM_LIMIT_MIB_FOR_DERIVATION = 16 * 1024;
|
|
16040
16101
|
HINDSIGHT_PG_SHARED_BUFFERS_BUDGET_MIB = HINDSIGHT_PG_MEM_LIMIT_MIB_FOR_DERIVATION - HINDSIGHT_PG_APP_ANON_MIB - HINDSIGHT_PG_PAGE_CACHE_FLOOR_MIB;
|
|
16102
|
+
HINDSIGHT_PG_MIN_NON_BUFFER_MIB = HINDSIGHT_PG_APP_ANON_MIB + HINDSIGHT_PG_PAGE_CACHE_FLOOR_MIB;
|
|
16041
16103
|
HINDSIGHT_PG_DEFAULTS = [
|
|
16042
16104
|
[
|
|
16043
16105
|
HINDSIGHT_PG_EFFECTIVE_CACHE_SIZE_ENV,
|
|
@@ -16128,7 +16190,7 @@ function hindsightPerfEnv(caps, overrides = new Map) {
|
|
|
16128
16190
|
function findUnmanagedHindsightEnvKeys(configEnv) {
|
|
16129
16191
|
return Object.keys(configEnv ?? {}).filter((key) => !HINDSIGHT_PERF_ENV_KEYS.has(key) && !HINDSIGHT_PG_ENV_KEYS.has(key)).sort();
|
|
16130
16192
|
}
|
|
16131
|
-
var HINDSIGHT_RERANKER_MAX_CANDIDATES_FOR_DERIVATION = 150, HINDSIGHT_DEFAULT_RECALL_MAX_CANDIDATES_PER_SOURCE, HINDSIGHT_DEFAULT_LINK_EXPANSION_PER_ENTITY_LIMIT = 50, HINDSIGHT_DEFAULT_LINK_EXPANSION_TIMEOUT_S = 2, HINDSIGHT_DEFAULT_LLM_REASONING_EFFORT = "low", HINDSIGHT_DEFAULT_LLM_MAX_CONCURRENT = 4, HINDSIGHT_DEFAULT_RETAIN_LLM_MAX_CONCURRENT = 1, HINDSIGHT_DEFAULT_CONSOLIDATION_LLM_MAX_CONCURRENT, HINDSIGHT_DEFAULT_RERANKER_LOCAL_FP16 = "true", HINDSIGHT_DEFAULT_RERANKER_LOCAL_BATCH_SIZE = 128, HINDSIGHT_DEFAULT_LLM_STRICT_SCHEMA = "true", HINDSIGHT_DEFAULT_LLM_MAX_RETRIES = 2, HINDSIGHT_DEFAULT_CONSOLIDATION_LLM_PARALLELISM = 2, HINDSIGHT_DEFAULT_MAX_OBSERVATIONS_PER_SCOPE = 1000, HINDSIGHT_DEFAULT_RERANKER_BUCKET_BATCHING = "true", HINDSIGHT_DEFAULT_RERANKER_MAX_CANDIDATES = 150, HINDSIGHT_DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT = 4, HINDSIGHT_DEFAULT_RECALL_MAX_CONCURRENT = 8, HINDSIGHT_DEFAULT_REFLECT_WALL_TIMEOUT_S = 600, HINDSIGHT_DEFAULT_CONSOLIDATION_MAX_MEMORIES_PER_ROUND = 250, HINDSIGHT_DEFAULT_CONSOLIDATION_SLOT_LIMIT = 6, HINDSIGHT_DEFAULT_CONSOLIDATION_RESERVED_SLOTS = 1, HINDSIGHT_DEFAULT_GRAPH_SEED_MIN_SIMILARITY = 0.3, HINDSIGHT_DEFAULT_LLM_SUPPORTS_MAX_ITEMS = "false", HINDSIGHT_DEFAULT_RECENCY_DECAY_FUNCTION = "exponential", HINDSIGHT_DEFAULT_RECENCY_DECAY_HALFLIFE_DAYS = 30, HINDSIGHT_PERF_DEFAULTS_UNGATED, HINDSIGHT_PERF_DEFAULTS_GPU, HINDSIGHT_PERF_DEFAULTS_LOCAL_LLM, HINDSIGHT_PERF_OVERRIDE_ONLY_KEYS, HINDSIGHT_WORKER_SLOT_TYPES, HINDSIGHT_WORKER_RESERVED_SLOT_ALIASES, HINDSIGHT_PERF_ENV_KEYS;
|
|
16193
|
+
var HINDSIGHT_RERANKER_MAX_CANDIDATES_FOR_DERIVATION = 150, HINDSIGHT_DEFAULT_RECALL_MAX_CANDIDATES_PER_SOURCE, HINDSIGHT_DEFAULT_LINK_EXPANSION_PER_ENTITY_LIMIT = 50, HINDSIGHT_DEFAULT_LINK_EXPANSION_TIMEOUT_S = 2, HINDSIGHT_DEFAULT_LLM_REASONING_EFFORT = "low", HINDSIGHT_DEFAULT_LLM_MAX_CONCURRENT = 4, HINDSIGHT_DEFAULT_RETAIN_LLM_MAX_CONCURRENT = 1, HINDSIGHT_DEFAULT_CONSOLIDATION_LLM_MAX_CONCURRENT, HINDSIGHT_DEFAULT_RERANKER_LOCAL_FP16 = "true", HINDSIGHT_DEFAULT_RERANKER_LOCAL_BATCH_SIZE = 128, HINDSIGHT_DEFAULT_LLM_STRICT_SCHEMA = "true", HINDSIGHT_DEFAULT_LLM_MAX_RETRIES = 2, HINDSIGHT_DEFAULT_CONSOLIDATION_LLM_PARALLELISM = 2, HINDSIGHT_DEFAULT_MAX_OBSERVATIONS_PER_SCOPE = 1000, HINDSIGHT_DEFAULT_RERANKER_BUCKET_BATCHING = "true", HINDSIGHT_DEFAULT_RERANKER_MAX_CANDIDATES = 150, HINDSIGHT_DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT = 4, HINDSIGHT_DEFAULT_RECALL_MAX_CONCURRENT = 8, HINDSIGHT_DEFAULT_REFLECT_WALL_TIMEOUT_S = 600, HINDSIGHT_DEFAULT_CONSOLIDATION_MAX_MEMORIES_PER_ROUND = 250, HINDSIGHT_DEFAULT_CONSOLIDATION_SLOT_LIMIT = 6, HINDSIGHT_DEFAULT_CONSOLIDATION_RESERVED_SLOTS = 1, HINDSIGHT_DEFAULT_GRAPH_SEED_MIN_SIMILARITY = 0.3, HINDSIGHT_DEFAULT_LLM_SUPPORTS_MAX_ITEMS = "false", HINDSIGHT_DEFAULT_RECENCY_DECAY_FUNCTION = "exponential", HINDSIGHT_DEFAULT_RECENCY_DECAY_HALFLIFE_DAYS = 30, HINDSIGHT_DEFAULT_TEXT_SEARCH_EXTENSION = "pg_search", HINDSIGHT_PERF_DEFAULTS_UNGATED, HINDSIGHT_PERF_DEFAULTS_GPU, HINDSIGHT_PERF_DEFAULTS_LOCAL_LLM, HINDSIGHT_PERF_OVERRIDE_ONLY_KEYS, HINDSIGHT_WORKER_SLOT_TYPES, HINDSIGHT_WORKER_RESERVED_SLOT_ALIASES, HINDSIGHT_PERF_ENV_KEYS;
|
|
16132
16194
|
var init_hindsight_perf_defaults = __esm(() => {
|
|
16133
16195
|
init_hindsight_pg_defaults();
|
|
16134
16196
|
HINDSIGHT_DEFAULT_RECALL_MAX_CANDIDATES_PER_SOURCE = Math.ceil(HINDSIGHT_RERANKER_MAX_CANDIDATES_FOR_DERIVATION * 0.4);
|
|
@@ -16198,7 +16260,8 @@ var init_hindsight_perf_defaults = __esm(() => {
|
|
|
16198
16260
|
[
|
|
16199
16261
|
"HINDSIGHT_API_GRAPH_SEED_MIN_SIMILARITY",
|
|
16200
16262
|
String(HINDSIGHT_DEFAULT_GRAPH_SEED_MIN_SIMILARITY)
|
|
16201
|
-
]
|
|
16263
|
+
],
|
|
16264
|
+
["HINDSIGHT_API_TEXT_SEARCH_EXTENSION", HINDSIGHT_DEFAULT_TEXT_SEARCH_EXTENSION]
|
|
16202
16265
|
];
|
|
16203
16266
|
HINDSIGHT_PERF_DEFAULTS_GPU = [
|
|
16204
16267
|
["HINDSIGHT_API_RERANKER_LOCAL_FP16", HINDSIGHT_DEFAULT_RERANKER_LOCAL_FP16],
|
|
@@ -18733,17 +18796,20 @@ function hindsightCpAuthEnvPairs(opts) {
|
|
|
18733
18796
|
return pairs;
|
|
18734
18797
|
}
|
|
18735
18798
|
async function resolveHindsightCpAccessKey(config, deps = {}) {
|
|
18736
|
-
|
|
18737
|
-
|
|
18799
|
+
return resolveHindsightVaultString(config?.hindsight?.cp_access_key, deps);
|
|
18800
|
+
}
|
|
18801
|
+
async function resolveHindsightVaultString(ref, deps = {}) {
|
|
18802
|
+
const trimmed = ref?.trim();
|
|
18803
|
+
if (!trimmed)
|
|
18738
18804
|
return;
|
|
18739
18805
|
const { isVaultReference: isVaultReference2, parseVaultReference: parseVaultReference2 } = await Promise.resolve().then(() => (init_resolver(), exports_resolver));
|
|
18740
|
-
if (!isVaultReference2(
|
|
18741
|
-
return
|
|
18806
|
+
if (!isVaultReference2(trimmed))
|
|
18807
|
+
return trimmed;
|
|
18742
18808
|
const brokerClient = await Promise.resolve().then(() => (init_client(), exports_client));
|
|
18743
18809
|
const get = deps.getViaBrokerStructured ?? brokerClient.getViaBrokerStructured;
|
|
18744
18810
|
const agentSlug = process.env.SWITCHROOM_AGENT_NAME;
|
|
18745
18811
|
const token = agentSlug ? brokerClient.readVaultTokenFile(agentSlug) ?? undefined : undefined;
|
|
18746
|
-
const resolved = await get(parseVaultReference2(
|
|
18812
|
+
const resolved = await get(parseVaultReference2(trimmed), {
|
|
18747
18813
|
...token ? { token } : {}
|
|
18748
18814
|
}).catch(() => null);
|
|
18749
18815
|
if (!resolved || resolved.kind !== "ok" || resolved.entry.kind !== "string") {
|
|
@@ -18752,6 +18818,64 @@ async function resolveHindsightCpAccessKey(config, deps = {}) {
|
|
|
18752
18818
|
const value = resolved.entry.value.trim();
|
|
18753
18819
|
return value.length > 0 ? value : undefined;
|
|
18754
18820
|
}
|
|
18821
|
+
async function resolveHindsightLiteLlm(config, deps = {}) {
|
|
18822
|
+
const top = config?.litellm;
|
|
18823
|
+
if (!top?.enabled || !top.base_url?.trim())
|
|
18824
|
+
return {};
|
|
18825
|
+
const ref = `vault:${HINDSIGHT_LITELLM_KEY_VAULT_REF}`;
|
|
18826
|
+
const apiKey = await resolveHindsightVaultString(ref, deps);
|
|
18827
|
+
if (!apiKey)
|
|
18828
|
+
return { droppedRef: ref };
|
|
18829
|
+
return {
|
|
18830
|
+
litellm: {
|
|
18831
|
+
baseUrl: top.base_url,
|
|
18832
|
+
apiKey,
|
|
18833
|
+
model: config?.memory?.config?.llm_model
|
|
18834
|
+
}
|
|
18835
|
+
};
|
|
18836
|
+
}
|
|
18837
|
+
function hindsightLiteLlmDroppedKeyWarning(ref) {
|
|
18838
|
+
return `hindsight: LiteLLM routing key \`${ref}\` did not resolve through the ` + "vault broker (denied, missing, or broker down) \u2014 ANTHROPIC_BASE_URL and " + "ANTHROPIC_CUSTOM_HEADERS were DROPPED, so hindsight's LLM traffic will bypass the proxy entirely and its spend will NOT be metered under `end_user=hindsight`. Check the broker grant for the reference above, then re-run. If an ENV DRIFT report above named those two vars, THIS is " + "why \u2014 it is NOT operator env drift.";
|
|
18839
|
+
}
|
|
18840
|
+
async function resolveHindsightLlmSecrets(llm, deps = {}) {
|
|
18841
|
+
if (!llm)
|
|
18842
|
+
return llm;
|
|
18843
|
+
const out = { ...llm };
|
|
18844
|
+
if (llm.api_key !== undefined) {
|
|
18845
|
+
out.api_key = await resolveHindsightVaultString(llm.api_key, deps);
|
|
18846
|
+
}
|
|
18847
|
+
for (const op of HINDSIGHT_LLM_OPS) {
|
|
18848
|
+
const cfg = llm[op];
|
|
18849
|
+
if (cfg?.api_key !== undefined) {
|
|
18850
|
+
out[op] = {
|
|
18851
|
+
...cfg,
|
|
18852
|
+
api_key: await resolveHindsightVaultString(cfg.api_key, deps)
|
|
18853
|
+
};
|
|
18854
|
+
}
|
|
18855
|
+
}
|
|
18856
|
+
return out;
|
|
18857
|
+
}
|
|
18858
|
+
async function diffDroppedHindsightLlmVaultKeys(original, resolved) {
|
|
18859
|
+
if (!original)
|
|
18860
|
+
return [];
|
|
18861
|
+
const { isVaultReference: isVaultReference2 } = await Promise.resolve().then(() => (init_resolver(), exports_resolver));
|
|
18862
|
+
const drops = [];
|
|
18863
|
+
const check = (lane, origKey, resolvedKey) => {
|
|
18864
|
+
const ref = origKey?.trim();
|
|
18865
|
+
if (ref && isVaultReference2(ref) && !resolvedKey)
|
|
18866
|
+
drops.push({ lane, ref });
|
|
18867
|
+
};
|
|
18868
|
+
check("global", original.api_key, resolved?.api_key);
|
|
18869
|
+
for (const op of HINDSIGHT_LLM_OPS) {
|
|
18870
|
+
check(op, original[op]?.api_key, resolved?.[op]?.api_key);
|
|
18871
|
+
}
|
|
18872
|
+
return drops;
|
|
18873
|
+
}
|
|
18874
|
+
function hindsightLlmDroppedKeyWarning(drop) {
|
|
18875
|
+
const where = drop.lane === "global" ? "the global LLM lane" : `the \`${drop.lane}\` LLM op`;
|
|
18876
|
+
const fallback = drop.lane === "global" ? "every op inherits" : "the op inherits";
|
|
18877
|
+
return `hindsight: ${where} api_key \`${drop.ref}\` did not resolve through the ` + "vault broker (denied, missing, or broker down) \u2014 the key was DROPPED, so " + `${fallback} the provider-default credential and fact extraction on that lane may fail. Check the broker grant for the reference above, then re-run.`;
|
|
18878
|
+
}
|
|
18755
18879
|
function hindsightLocalLlmEnabled(llm, litellm, override) {
|
|
18756
18880
|
if (override !== undefined)
|
|
18757
18881
|
return override;
|
|
@@ -18859,6 +18983,29 @@ function deviceRequestsHaveGpu(reqs) {
|
|
|
18859
18983
|
return (r.Count ?? 0) !== 0;
|
|
18860
18984
|
});
|
|
18861
18985
|
}
|
|
18986
|
+
function resolveHindsightMemLimit(perf) {
|
|
18987
|
+
const raw = perf?.memLimit?.trim();
|
|
18988
|
+
if (!raw)
|
|
18989
|
+
return HINDSIGHT_DEFAULT_MEM_LIMIT;
|
|
18990
|
+
const capMib = parseDockerSizeToMib(raw);
|
|
18991
|
+
if (capMib === null) {
|
|
18992
|
+
throw new Error(`hindsight.mem_limit: \`${raw}\` is not a docker memory size. Use a number with an optional b/k/m/g suffix, e.g. \`24g\` or \`16384m\`.`);
|
|
18993
|
+
}
|
|
18994
|
+
const reservationMib = parseDockerSizeToMib(HINDSIGHT_DEFAULT_MEM_RESERVATION);
|
|
18995
|
+
if (reservationMib !== null && capMib < reservationMib) {
|
|
18996
|
+
throw new Error(`hindsight.mem_limit: \`${raw}\` is below the container's memory reservation (${HINDSIGHT_DEFAULT_MEM_RESERVATION}) \u2014 docker refuses ` + "to create a container whose memory limit is under its reservation.");
|
|
18997
|
+
}
|
|
18998
|
+
return raw;
|
|
18999
|
+
}
|
|
19000
|
+
function hindsightMemBudgetWarningFor(perf) {
|
|
19001
|
+
const sharedBuffers = hindsightPgEnvPairs(perf).find(([k]) => k === HINDSIGHT_PG_SHARED_BUFFERS_ENV)?.[1];
|
|
19002
|
+
if (!sharedBuffers)
|
|
19003
|
+
return null;
|
|
19004
|
+
return hindsightMemBudgetWarning({
|
|
19005
|
+
memLimit: resolveHindsightMemLimit(perf),
|
|
19006
|
+
sharedBuffers
|
|
19007
|
+
});
|
|
19008
|
+
}
|
|
18862
19009
|
function hindsightPerfEnvPairs(llm, litellm, gpu, perf) {
|
|
18863
19010
|
return hindsightPerfEnv({
|
|
18864
19011
|
gpu: hindsightGpuEnabled(gpu),
|
|
@@ -18868,6 +19015,23 @@ function hindsightPerfEnvPairs(llm, litellm, gpu, perf) {
|
|
|
18868
19015
|
function hindsightPgEnvPairs(perf) {
|
|
18869
19016
|
return hindsightPgEnv(resolveHindsightPgOverrides(perf?.env, perf?.processEnv));
|
|
18870
19017
|
}
|
|
19018
|
+
function hindsightLiteLlmEnvPairs(litellm, llmModel) {
|
|
19019
|
+
const baseUrl = litellm?.baseUrl?.trim();
|
|
19020
|
+
const apiKey = litellm?.apiKey?.trim();
|
|
19021
|
+
if (!baseUrl || !apiKey)
|
|
19022
|
+
return [];
|
|
19023
|
+
const litellmRoot = baseUrl.replace(/\/+$/, "");
|
|
19024
|
+
const anthropicBaseUrl = isClaudeModel(llmModel) ? `${litellmRoot}/anthropic` : litellmRoot;
|
|
19025
|
+
return [
|
|
19026
|
+
["ANTHROPIC_BASE_URL", anthropicBaseUrl],
|
|
19027
|
+
[
|
|
19028
|
+
"ANTHROPIC_CUSTOM_HEADERS",
|
|
19029
|
+
`x-litellm-api-key: Bearer ${apiKey}
|
|
19030
|
+
x-litellm-customer-id: ${HINDSIGHT_LITELLM_CUSTOMER_ID}
|
|
19031
|
+
x-litellm-tags: service:${HINDSIGHT_LITELLM_CUSTOMER_ID}`
|
|
19032
|
+
]
|
|
19033
|
+
];
|
|
19034
|
+
}
|
|
18871
19035
|
function hindsightContainerEnvPairs(opts) {
|
|
18872
19036
|
const { apiPort, litellm, llm, gpu, perf, cpAccessKey } = opts;
|
|
18873
19037
|
const { provider: llmProvider, model: llmModel } = resolveHindsightLlm(llm, litellm);
|
|
@@ -18895,16 +19059,7 @@ function hindsightContainerEnvPairs(opts) {
|
|
|
18895
19059
|
accessKey: cpAccessKey,
|
|
18896
19060
|
hostNetwork: hindsightNeedsHostNetwork(llm, litellm)
|
|
18897
19061
|
}));
|
|
18898
|
-
|
|
18899
|
-
const litellmRoot = litellm.baseUrl.replace(/\/+$/, "");
|
|
18900
|
-
const anthropicBaseUrl = isClaudeModel(llmModel) ? `${litellmRoot}/anthropic` : litellmRoot;
|
|
18901
|
-
pairs.push(["ANTHROPIC_BASE_URL", anthropicBaseUrl], [
|
|
18902
|
-
"ANTHROPIC_CUSTOM_HEADERS",
|
|
18903
|
-
`x-litellm-api-key: Bearer ${litellm.apiKey}
|
|
18904
|
-
x-litellm-customer-id: hindsight
|
|
18905
|
-
x-litellm-tags: service:hindsight`
|
|
18906
|
-
]);
|
|
18907
|
-
}
|
|
19062
|
+
pairs.push(...hindsightLiteLlmEnvPairs(litellm, llmModel));
|
|
18908
19063
|
return pairs;
|
|
18909
19064
|
}
|
|
18910
19065
|
function startHindsight(ports, litellm, imageTag, llm, mirrorDir, gpu, perf, cpAccessKey) {
|
|
@@ -18918,6 +19073,9 @@ function startHindsight(ports, litellm, imageTag, llm, mirrorDir, gpu, perf, cpA
|
|
|
18918
19073
|
gpu,
|
|
18919
19074
|
perf
|
|
18920
19075
|
}).flatMap(([k, v]) => ["-e", `${k}=${v}`]);
|
|
19076
|
+
const memBudgetWarning = hindsightMemBudgetWarningFor(perf);
|
|
19077
|
+
if (memBudgetWarning)
|
|
19078
|
+
console.warn(` ! ${memBudgetWarning}`);
|
|
18921
19079
|
const args = [
|
|
18922
19080
|
"run",
|
|
18923
19081
|
"-d",
|
|
@@ -18925,7 +19083,7 @@ function startHindsight(ports, litellm, imageTag, llm, mirrorDir, gpu, perf, cpA
|
|
|
18925
19083
|
"switchroom-hindsight",
|
|
18926
19084
|
"--restart",
|
|
18927
19085
|
"always",
|
|
18928
|
-
`--memory=${
|
|
19086
|
+
`--memory=${resolveHindsightMemLimit(perf)}`,
|
|
18929
19087
|
`--memory-reservation=${HINDSIGHT_DEFAULT_MEM_RESERVATION}`,
|
|
18930
19088
|
`--pids-limit=${HINDSIGHT_DEFAULT_PIDS_LIMIT}`,
|
|
18931
19089
|
`--shm-size=${HINDSIGHT_DEFAULT_SHM_SIZE}`,
|
|
@@ -19111,11 +19269,7 @@ function generateHindsightComposeSnippet(llm, mirrorDir, litellm, gpu, perf, cpA
|
|
|
19111
19269
|
...hostNetwork ? hindsightHostNetworkBindEnvPairs(apiPort).map(([k, v]) => ` - ${k}=${v}`) : [` - HINDSIGHT_CP_DATAPLANE_API_URL=http://localhost:8888`],
|
|
19112
19270
|
...hindsightCpAuthEnvPairs({ accessKey: cpAccessKey, hostNetwork }).map(([k, v]) => ` - ${k}=${v}`)
|
|
19113
19271
|
];
|
|
19114
|
-
|
|
19115
|
-
const litellmRoot = litellm.baseUrl.replace(/\/+$/, "");
|
|
19116
|
-
const anthropicBaseUrl = isClaudeModel(llmModel) ? `${litellmRoot}/anthropic` : litellmRoot;
|
|
19117
|
-
environment.push(` - ANTHROPIC_BASE_URL=${anthropicBaseUrl}`, ` - ANTHROPIC_CUSTOM_HEADERS=x-litellm-api-key: Bearer ${litellm.apiKey}\\nx-litellm-customer-id: hindsight\\nx-litellm-tags: service:hindsight`);
|
|
19118
|
-
}
|
|
19272
|
+
environment.push(...hindsightLiteLlmEnvPairs(litellm, llmModel).map(([k, v]) => ` - ${k}=${v.replace(/\n/g, "\\n")}`));
|
|
19119
19273
|
const initService = mirrorDir ? [
|
|
19120
19274
|
" switchroom-hindsight-creds-init:",
|
|
19121
19275
|
` image: ${HINDSIGHT_IMAGE}`,
|
|
@@ -19156,7 +19310,7 @@ function generateHindsightComposeSnippet(llm, mirrorDir, litellm, gpu, perf, cpA
|
|
|
19156
19310
|
...hindsightPgEnvPairs(perf).map(([k, v]) => ` - ${k}=${v}`),
|
|
19157
19311
|
` - SWITCHROOM_HINDSIGHT_REQUEUE_DEAD_LETTERS=${HINDSIGHT_DEFAULT_REQUEUE_DEAD_LETTERS}`,
|
|
19158
19312
|
` - SWITCHROOM_HINDSIGHT_REQUEUE_MAX=${HINDSIGHT_DEFAULT_REQUEUE_MAX}`,
|
|
19159
|
-
` mem_limit: ${
|
|
19313
|
+
` mem_limit: ${resolveHindsightMemLimit(perf)}`,
|
|
19160
19314
|
` mem_reservation: ${HINDSIGHT_DEFAULT_MEM_RESERVATION}`,
|
|
19161
19315
|
` pids_limit: ${HINDSIGHT_DEFAULT_PIDS_LIMIT}`,
|
|
19162
19316
|
` shm_size: ${HINDSIGHT_DEFAULT_SHM_SIZE}`,
|
|
@@ -19265,7 +19419,7 @@ var HINDSIGHT_DEFAULT_API_PORT = 18888, HINDSIGHT_DEFAULT_UI_PORT = 9999, HINDSI
|
|
|
19265
19419
|
} catch {
|
|
19266
19420
|
return null;
|
|
19267
19421
|
}
|
|
19268
|
-
}, HINDSIGHT_LLM_OPS, HINDSIGHT_HOST_NETWORK_BIND_ADDR = "127.0.0.1", HINDSIGHT_CP_UNAUTHENTICATED_BIND_ADDR = "127.0.0.1", HINDSIGHT_CP_AUTHENTICATED_BIND_ADDR = "0.0.0.0", HINDSIGHT_CP_NO_ACCESS_KEY_WARNING;
|
|
19422
|
+
}, HINDSIGHT_LLM_OPS, HINDSIGHT_HOST_NETWORK_BIND_ADDR = "127.0.0.1", HINDSIGHT_CP_UNAUTHENTICATED_BIND_ADDR = "127.0.0.1", HINDSIGHT_CP_AUTHENTICATED_BIND_ADDR = "0.0.0.0", HINDSIGHT_CP_NO_ACCESS_KEY_WARNING, HINDSIGHT_LITELLM_KEY_VAULT_REF = "litellm/hindsight/api-key", HINDSIGHT_LITELLM_CUSTOMER_ID = "hindsight";
|
|
19269
19423
|
var init_hindsight = __esm(() => {
|
|
19270
19424
|
init_model_command();
|
|
19271
19425
|
init_timeout_budget();
|
|
@@ -32857,8 +33011,8 @@ function parseMemToMib(s) {
|
|
|
32857
33011
|
return Math.round(n * mult);
|
|
32858
33012
|
}
|
|
32859
33013
|
function bumpMem(s, addMib) {
|
|
32860
|
-
const
|
|
32861
|
-
return
|
|
33014
|
+
const mib2 = parseMemToMib(s);
|
|
33015
|
+
return mib2 == null ? s : `${mib2 + addMib}m`;
|
|
32862
33016
|
}
|
|
32863
33017
|
function resolveResourceDefaults(agentName, profile, overrides, opts) {
|
|
32864
33018
|
let base;
|
|
@@ -47577,12 +47731,12 @@ function classifyDirectiveCount(count, bankLabel) {
|
|
|
47577
47731
|
return null;
|
|
47578
47732
|
}
|
|
47579
47733
|
function classifyShmSize(bytes) {
|
|
47580
|
-
const
|
|
47734
|
+
const mib2 = Math.round(bytes / 1024 / 1024);
|
|
47581
47735
|
if (bytes < MIN_HINDSIGHT_SHM_BYTES) {
|
|
47582
47736
|
return {
|
|
47583
47737
|
name: "hindsight shm-size",
|
|
47584
47738
|
status: "fail",
|
|
47585
|
-
detail: `${
|
|
47739
|
+
detail: `${mib2}MB \u2014 PostgreSQL needs ~533MB+ for shared segments; writes will ` + `fail with "No space left on device"`,
|
|
47586
47740
|
fix: "Recreate hindsight with a larger shm. The launch path now sets " + "--shm-size=2g (#2190); pull a release that includes it and run " + "`switchroom memory --restart`, or recreate the container manually " + "preserving the `switchroom-hindsight-data` volume."
|
|
47587
47741
|
};
|
|
47588
47742
|
}
|
|
@@ -106656,7 +106810,7 @@ function formatDroppedHindsightEnvReport(dropped, caps) {
|
|
|
106656
106810
|
|
|
106657
106811
|
// src/cli/memory.ts
|
|
106658
106812
|
init_hindsight_repair();
|
|
106659
|
-
|
|
106813
|
+
init_resolver();
|
|
106660
106814
|
init_atomic();
|
|
106661
106815
|
init_loader();
|
|
106662
106816
|
init_hindsight();
|
|
@@ -106706,22 +106860,32 @@ function resolveMemorySetupTag(input) {
|
|
|
106706
106860
|
return { tag: pin, reason: "pin" };
|
|
106707
106861
|
return { tag: undefined, reason: "latest" };
|
|
106708
106862
|
}
|
|
106709
|
-
|
|
106710
|
-
const
|
|
106711
|
-
|
|
106712
|
-
|
|
106713
|
-
|
|
106714
|
-
const token = agentSlug ? readVaultTokenFile(agentSlug) ?? undefined : undefined;
|
|
106715
|
-
const result = await getViaBrokerStructured("litellm/hindsight/api-key", {
|
|
106716
|
-
...token ? { token } : {}
|
|
106717
|
-
}).catch(() => null);
|
|
106718
|
-
if (!result || result.kind !== "ok" || result.entry.kind !== "string")
|
|
106719
|
-
return;
|
|
106720
|
-
return {
|
|
106721
|
-
baseUrl: topLiteLLM.base_url,
|
|
106722
|
-
apiKey: result.entry.value,
|
|
106723
|
-
model: config.memory?.config?.llm_model
|
|
106863
|
+
function hindsightSecretValues(input) {
|
|
106864
|
+
const out = [];
|
|
106865
|
+
const push = (v) => {
|
|
106866
|
+
if (typeof v === "string" && v.trim().length > 0)
|
|
106867
|
+
out.push(v.trim());
|
|
106724
106868
|
};
|
|
106869
|
+
push(input.cpAccessKey);
|
|
106870
|
+
push(input.llm?.api_key);
|
|
106871
|
+
for (const op of ["retain", "reflect", "consolidation"]) {
|
|
106872
|
+
push(input.llm?.[op]?.api_key);
|
|
106873
|
+
}
|
|
106874
|
+
return out;
|
|
106875
|
+
}
|
|
106876
|
+
function hasUnresolvedHindsightVaultRef(config) {
|
|
106877
|
+
const hindsight = config.hindsight;
|
|
106878
|
+
return hindsightSecretValues({
|
|
106879
|
+
llm: hindsight?.llm,
|
|
106880
|
+
cpAccessKey: hindsight?.cp_access_key
|
|
106881
|
+
}).some((v) => isVaultReference(v));
|
|
106882
|
+
}
|
|
106883
|
+
function emitsCleartextHindsightSecret(input) {
|
|
106884
|
+
return hindsightSecretValues(input).some((v) => !isVaultReference(v));
|
|
106885
|
+
}
|
|
106886
|
+
var HINDSIGHT_COMPOSE_CLEARTEXT_SECRETS_WARNING = "This snippet embeds secrets in CLEARTEXT (LLM api_key and/or cp_access_key)." + " Treat the output as sensitive \u2014 do not paste it into shared channels or" + " commit it unredacted.";
|
|
106887
|
+
async function resolveLiteLLMForHindsight(config) {
|
|
106888
|
+
return resolveHindsightLiteLlm(config);
|
|
106725
106889
|
}
|
|
106726
106890
|
function readRecallLog(agentDir, limit) {
|
|
106727
106891
|
const path7 = join80(agentDir, ".claude", "plugins", "data", "hindsight-memory-inline", "state", "recall_log.jsonl");
|
|
@@ -106933,6 +107097,13 @@ Cross-agent reflection plan
|
|
|
106933
107097
|
process.exit(1);
|
|
106934
107098
|
}
|
|
106935
107099
|
const recreate = opts.recreate === true;
|
|
107100
|
+
let resolvedLlmPromise;
|
|
107101
|
+
const getResolvedLlm = () => {
|
|
107102
|
+
if (!resolvedLlmPromise) {
|
|
107103
|
+
resolvedLlmPromise = resolveHindsightLlmSecrets(getConfig(program3).hindsight?.llm);
|
|
107104
|
+
}
|
|
107105
|
+
return resolvedLlmPromise;
|
|
107106
|
+
};
|
|
106936
107107
|
let releasePin;
|
|
106937
107108
|
try {
|
|
106938
107109
|
releasePin = getConfig(program3).release?.pin ?? undefined;
|
|
@@ -107001,18 +107172,19 @@ Cross-agent reflection plan
|
|
|
107001
107172
|
const liveEnv = getHindsightContainerEnv();
|
|
107002
107173
|
if (liveEnv) {
|
|
107003
107174
|
const driftConfig = getConfig(program3);
|
|
107004
|
-
const driftLitellm = await resolveLiteLLMForHindsight(driftConfig);
|
|
107175
|
+
const driftLitellm = (await resolveLiteLLMForHindsight(driftConfig)).litellm;
|
|
107005
107176
|
const driftPerf = { env: driftConfig.hindsight?.env };
|
|
107177
|
+
const driftLlm = await getResolvedLlm();
|
|
107006
107178
|
const nextEnv = hindsightContainerEnvPairs({
|
|
107007
107179
|
apiPort: reusePorts?.apiPort ?? HINDSIGHT_DEFAULT_API_PORT,
|
|
107008
107180
|
litellm: driftLitellm,
|
|
107009
|
-
llm:
|
|
107181
|
+
llm: driftLlm,
|
|
107010
107182
|
gpu: gpuDecision.enabled,
|
|
107011
107183
|
perf: driftPerf,
|
|
107012
107184
|
cpAccessKey: await resolveHindsightCpAccessKey(driftConfig)
|
|
107013
107185
|
});
|
|
107014
107186
|
const dropped = diffDroppedHindsightEnv(liveEnv, nextEnv, getHindsightImageEnv() ?? []);
|
|
107015
|
-
envDriftLines = formatDroppedHindsightEnvReport(dropped, hindsightResolvedCapabilities(
|
|
107187
|
+
envDriftLines = formatDroppedHindsightEnvReport(dropped, hindsightResolvedCapabilities(driftLlm, driftLitellm, gpuDecision.enabled, driftPerf));
|
|
107016
107188
|
}
|
|
107017
107189
|
} catch (err) {
|
|
107018
107190
|
console.log(source_default.gray(` (env-drift check skipped: ${err.message})`));
|
|
@@ -107115,11 +107287,21 @@ Cross-agent reflection plan
|
|
|
107115
107287
|
console.log(source_default.gray(" Starting Hindsight Docker container..."));
|
|
107116
107288
|
try {
|
|
107117
107289
|
const hindsightConfig = getConfig(program3);
|
|
107118
|
-
const litellmCfg = await resolveLiteLLMForHindsight(hindsightConfig);
|
|
107290
|
+
const { litellm: litellmCfg, droppedRef: litellmDroppedRef } = await resolveLiteLLMForHindsight(hindsightConfig);
|
|
107291
|
+
if (litellmDroppedRef) {
|
|
107292
|
+
console.log(source_default.yellow(` ! ${hindsightLiteLlmDroppedKeyWarning(litellmDroppedRef)}`));
|
|
107293
|
+
}
|
|
107119
107294
|
const cpAccessKey = await resolveHindsightCpAccessKey(hindsightConfig);
|
|
107120
107295
|
if (!cpAccessKey)
|
|
107121
107296
|
console.log(source_default.yellow(` ! ${HINDSIGHT_CP_NO_ACCESS_KEY_WARNING}`));
|
|
107122
|
-
|
|
107297
|
+
const resolvedLlm = await getResolvedLlm();
|
|
107298
|
+
for (const drop of await diffDroppedHindsightLlmVaultKeys(hindsightConfig.hindsight?.llm, resolvedLlm)) {
|
|
107299
|
+
console.log(source_default.yellow(` ! ${hindsightLlmDroppedKeyWarning(drop)}`));
|
|
107300
|
+
}
|
|
107301
|
+
startHindsight(ports, litellmCfg, effectiveTag, resolvedLlm, hindsightConsumerMirrorDir(hindsightConfig), gpuDecision.enabled, {
|
|
107302
|
+
env: hindsightConfig.hindsight?.env,
|
|
107303
|
+
memLimit: hindsightConfig.hindsight?.mem_limit
|
|
107304
|
+
}, cpAccessKey);
|
|
107123
107305
|
if (litellmCfg) {
|
|
107124
107306
|
console.log(source_default.gray(" LiteLLM routing enabled for hindsight (--network host)."));
|
|
107125
107307
|
}
|
|
@@ -107144,17 +107326,47 @@ Cross-agent reflection plan
|
|
|
107144
107326
|
` + ` Add memory.config.url: ${url} manually.`));
|
|
107145
107327
|
}
|
|
107146
107328
|
});
|
|
107147
|
-
memory.command("docker-compose").description("Output a docker-compose snippet for Hindsight (broker-fed mode)").action(withConfigError(async () => {
|
|
107329
|
+
memory.command("docker-compose").description("Output a docker-compose snippet for Hindsight (broker-fed mode)").option("--resolve-secrets", "Resolve `vault:` refs (the LLM api_key fields and cp_access_key) to their " + "live secret values before printing the snippet. Without this flag (the " + "default) the `vault:` refs are printed literally \u2014 this is an " + "operator-invoked command whose stdout routinely ends up pasted into " + "terminals, files, and chat, and a live `sk-\u2026` key has no business " + "landing there by accident (#4487). Pass this when you genuinely want a " + "runnable compose file, and treat the output as a secret once you do.").action(withConfigError(async (opts) => {
|
|
107148
107330
|
console.log(source_default.bold(`
|
|
107149
107331
|
# Add this to your docker-compose.yml:
|
|
107150
107332
|
`));
|
|
107151
107333
|
{
|
|
107152
107334
|
const snippetConfig = getConfig(program3);
|
|
107153
|
-
const
|
|
107154
|
-
|
|
107155
|
-
|
|
107335
|
+
const resolveSecrets = opts.resolveSecrets === true;
|
|
107336
|
+
let cpAccessKey;
|
|
107337
|
+
let snippetLlm;
|
|
107338
|
+
if (resolveSecrets) {
|
|
107339
|
+
cpAccessKey = await resolveHindsightCpAccessKey(snippetConfig);
|
|
107340
|
+
if (!cpAccessKey) {
|
|
107341
|
+
console.error(source_default.yellow(`# ! ${HINDSIGHT_CP_NO_ACCESS_KEY_WARNING}`));
|
|
107342
|
+
}
|
|
107343
|
+
snippetLlm = await resolveHindsightLlmSecrets(snippetConfig.hindsight?.llm);
|
|
107344
|
+
} else {
|
|
107345
|
+
cpAccessKey = snippetConfig.hindsight?.cp_access_key;
|
|
107346
|
+
snippetLlm = snippetConfig.hindsight?.llm;
|
|
107347
|
+
if (hasUnresolvedHindsightVaultRef(snippetConfig)) {
|
|
107348
|
+
console.log(source_default.yellow("# NOTE: `vault:` references below are left UNRESOLVED (default) \u2014 this\n" + `# snippet is not runnable as-is. Resolve them yourself before use, or
|
|
107349
|
+
` + `# re-run with --resolve-secrets to print the live secret values
|
|
107350
|
+
` + `# instead. If you do, treat this command's output as a secret: don't
|
|
107351
|
+
` + `# paste it into chat, logs, or an unencrypted file.
|
|
107352
|
+
`));
|
|
107353
|
+
}
|
|
107156
107354
|
}
|
|
107157
|
-
|
|
107355
|
+
{
|
|
107356
|
+
const memWarning = hindsightMemBudgetWarningFor({
|
|
107357
|
+
env: snippetConfig.hindsight?.env,
|
|
107358
|
+
memLimit: snippetConfig.hindsight?.mem_limit
|
|
107359
|
+
});
|
|
107360
|
+
if (memWarning)
|
|
107361
|
+
console.error(source_default.yellow(`# ! ${memWarning}`));
|
|
107362
|
+
}
|
|
107363
|
+
if (emitsCleartextHindsightSecret({ llm: snippetLlm, cpAccessKey })) {
|
|
107364
|
+
console.error(source_default.yellow(`# ! ${HINDSIGHT_COMPOSE_CLEARTEXT_SECRETS_WARNING}`));
|
|
107365
|
+
}
|
|
107366
|
+
console.log(generateHindsightComposeSnippet(snippetLlm, hindsightConsumerMirrorDir(snippetConfig), undefined, hindsightGpuDecision(resolveHindsightGpuOverride({ config: snippetConfig.hindsight?.gpu })).enabled, {
|
|
107367
|
+
env: snippetConfig.hindsight?.env,
|
|
107368
|
+
memLimit: snippetConfig.hindsight?.mem_limit
|
|
107369
|
+
}, cpAccessKey));
|
|
107158
107370
|
}
|
|
107159
107371
|
console.log();
|
|
107160
107372
|
}));
|
|
@@ -111810,7 +112022,6 @@ init_onboarding();
|
|
|
111810
112022
|
init_gpu_detect();
|
|
111811
112023
|
init_host_capabilities();
|
|
111812
112024
|
init_hindsight();
|
|
111813
|
-
init_client();
|
|
111814
112025
|
init_posthog();
|
|
111815
112026
|
|
|
111816
112027
|
// src/cli/setup-posture-rewrite.ts
|
|
@@ -112634,22 +112845,8 @@ async function stepCreateTopics(config, botToken, nonInteractive) {
|
|
|
112634
112845
|
}
|
|
112635
112846
|
}
|
|
112636
112847
|
}
|
|
112637
|
-
async function resolveLiteLLMForHindsight2(config) {
|
|
112638
|
-
|
|
112639
|
-
if (!topLiteLLM?.enabled || !topLiteLLM.base_url)
|
|
112640
|
-
return;
|
|
112641
|
-
const agentSlug = process.env.SWITCHROOM_AGENT_NAME;
|
|
112642
|
-
const token = agentSlug ? readVaultTokenFile(agentSlug) ?? undefined : undefined;
|
|
112643
|
-
const result = await getViaBrokerStructured("litellm/hindsight/api-key", {
|
|
112644
|
-
...token ? { token } : {}
|
|
112645
|
-
}).catch(() => null);
|
|
112646
|
-
if (!result || result.kind !== "ok" || result.entry.kind !== "string")
|
|
112647
|
-
return;
|
|
112648
|
-
return {
|
|
112649
|
-
baseUrl: topLiteLLM.base_url,
|
|
112650
|
-
apiKey: result.entry.value,
|
|
112651
|
-
model: config.memory?.config?.llm_model
|
|
112652
|
-
};
|
|
112848
|
+
async function resolveLiteLLMForHindsight2(config, deps = {}) {
|
|
112849
|
+
return resolveHindsightLiteLlm(config, deps);
|
|
112653
112850
|
}
|
|
112654
112851
|
function isMemoryBackendDisabled(config) {
|
|
112655
112852
|
const memoryBackend = config.memory?.backend ?? "hindsight";
|
|
@@ -112748,13 +112945,20 @@ async function stepMemoryBackend(config, nonInteractive, switchroomConfigPath, d
|
|
|
112748
112945
|
}
|
|
112749
112946
|
const spin = spinner("Starting Hindsight Docker container...");
|
|
112750
112947
|
try {
|
|
112751
|
-
const litellmCfg = await resolveLiteLLMForHindsight2(config);
|
|
112948
|
+
const { litellm: litellmCfg, droppedRef: litellmDroppedRef } = await resolveLiteLLMForHindsight2(config, deps.getViaBrokerStructured ? { getViaBrokerStructured: deps.getViaBrokerStructured } : {});
|
|
112949
|
+
if (litellmDroppedRef) {
|
|
112950
|
+
console.log(source_default.yellow(` ! ${hindsightLiteLlmDroppedKeyWarning(litellmDroppedRef)}`));
|
|
112951
|
+
}
|
|
112752
112952
|
const cpAccessKey = await resolveHindsightCpAccessKey(config);
|
|
112753
112953
|
if (!cpAccessKey) {
|
|
112754
112954
|
console.log(source_default.yellow(` ! ${HINDSIGHT_CP_NO_ACCESS_KEY_WARNING}`));
|
|
112755
112955
|
}
|
|
112956
|
+
const resolvedLlm = await resolveHindsightLlmSecrets(config.hindsight?.llm, deps.getViaBrokerStructured ? { getViaBrokerStructured: deps.getViaBrokerStructured } : {});
|
|
112957
|
+
for (const drop of await diffDroppedHindsightLlmVaultKeys(config.hindsight?.llm, resolvedLlm)) {
|
|
112958
|
+
console.log(source_default.yellow(` ! ${hindsightLlmDroppedKeyWarning(drop)}`));
|
|
112959
|
+
}
|
|
112756
112960
|
const startContainer = deps.startContainer ?? startHindsight;
|
|
112757
|
-
startContainer(ports, litellmCfg, undefined,
|
|
112961
|
+
startContainer(ports, litellmCfg, undefined, resolvedLlm, hindsightConsumerMirrorDir(config), undefined, { env: config.hindsight?.env, memLimit: config.hindsight?.mem_limit }, cpAccessKey);
|
|
112758
112962
|
if (litellmCfg) {
|
|
112759
112963
|
console.log(source_default.gray(" LiteLLM routing enabled (--network host, ANTHROPIC_BASE_URL set)."));
|
|
112760
112964
|
}
|
|
@@ -128564,15 +128768,1193 @@ async function notifyOperator(agentsDir, text, log) {
|
|
|
128564
128768
|
return agent !== null;
|
|
128565
128769
|
}
|
|
128566
128770
|
|
|
128771
|
+
// src/cli/hindsight-bench.ts
|
|
128772
|
+
init_source();
|
|
128773
|
+
import { mkdirSync as mkdirSync71, readFileSync as readFileSync113, writeFileSync as writeFileSync52 } from "node:fs";
|
|
128774
|
+
import { dirname as dirname51, resolve as resolve67 } from "node:path";
|
|
128775
|
+
|
|
128776
|
+
// src/hindsight-bench/anonymise.ts
|
|
128777
|
+
function pseudonym(n, width) {
|
|
128778
|
+
return `bank-${String(n).padStart(width, "0")}`;
|
|
128779
|
+
}
|
|
128780
|
+
function buildBankMap(result) {
|
|
128781
|
+
const ordered = [];
|
|
128782
|
+
const seen = new Set;
|
|
128783
|
+
const push = (b) => {
|
|
128784
|
+
if (b !== "" && !seen.has(b)) {
|
|
128785
|
+
seen.add(b);
|
|
128786
|
+
ordered.push(b);
|
|
128787
|
+
}
|
|
128788
|
+
};
|
|
128789
|
+
for (const r of result.db?.bankRows ?? [])
|
|
128790
|
+
push(r.bank);
|
|
128791
|
+
for (const b of result.config?.banks ?? [])
|
|
128792
|
+
push(b);
|
|
128793
|
+
for (const c of result.cells ?? [])
|
|
128794
|
+
push(c.bank);
|
|
128795
|
+
for (const a of result.arms ?? [])
|
|
128796
|
+
push(a.bank);
|
|
128797
|
+
const width = Math.max(2, String(ordered.length).length);
|
|
128798
|
+
const map2 = new Map;
|
|
128799
|
+
ordered.forEach((bank, i2) => map2.set(bank, pseudonym(i2 + 1, width)));
|
|
128800
|
+
return map2;
|
|
128801
|
+
}
|
|
128802
|
+
function anonymiseResult(result) {
|
|
128803
|
+
const mapping = buildBankMap(result);
|
|
128804
|
+
const sub = (b) => mapping.get(b) ?? b;
|
|
128805
|
+
return {
|
|
128806
|
+
mapping,
|
|
128807
|
+
result: {
|
|
128808
|
+
...result,
|
|
128809
|
+
config: { ...result.config, banks: (result.config?.banks ?? []).map(sub) },
|
|
128810
|
+
db: {
|
|
128811
|
+
...result.db,
|
|
128812
|
+
bankRows: (result.db?.bankRows ?? []).map((r) => ({ ...r, bank: sub(r.bank) }))
|
|
128813
|
+
},
|
|
128814
|
+
cells: (result.cells ?? []).map((c) => ({ ...c, bank: sub(c.bank) })),
|
|
128815
|
+
arms: result.arms === null ? null : result.arms.map((a) => ({ ...a, bank: sub(a.bank) }))
|
|
128816
|
+
}
|
|
128817
|
+
};
|
|
128818
|
+
}
|
|
128819
|
+
function formatBankMapping(mapping) {
|
|
128820
|
+
return [...mapping.entries()].map(([real, pseudo]) => ` ${pseudo} ${real}`).join(`
|
|
128821
|
+
`);
|
|
128822
|
+
}
|
|
128823
|
+
|
|
128824
|
+
// src/hindsight-bench/banks.ts
|
|
128825
|
+
class BankSelectionError extends Error {
|
|
128826
|
+
}
|
|
128827
|
+
function selectBanks(available, spec) {
|
|
128828
|
+
const nonEmpty = available.filter((b) => b.rows > 0).sort((a, b) => b.rows - a.rows);
|
|
128829
|
+
const trimmed = spec.trim();
|
|
128830
|
+
if (trimmed === "all")
|
|
128831
|
+
return nonEmpty.map((b) => b.bank);
|
|
128832
|
+
const top = /^top:(\d+)$/.exec(trimmed);
|
|
128833
|
+
if (top !== null)
|
|
128834
|
+
return nonEmpty.slice(0, Math.max(1, Number(top[1]))).map((b) => b.bank);
|
|
128835
|
+
const spread = /^spread:(\d+)$/.exec(trimmed);
|
|
128836
|
+
if (spread !== null) {
|
|
128837
|
+
const n = Math.max(1, Number(spread[1]));
|
|
128838
|
+
if (nonEmpty.length <= n)
|
|
128839
|
+
return nonEmpty.map((b) => b.bank);
|
|
128840
|
+
const asc = [...nonEmpty].sort((a, b) => a.rows - b.rows);
|
|
128841
|
+
const lo = Math.log10(asc[0].rows);
|
|
128842
|
+
const hi = Math.log10(asc[asc.length - 1].rows);
|
|
128843
|
+
const chosen = new Set;
|
|
128844
|
+
for (let i2 = 0;i2 < n; i2++) {
|
|
128845
|
+
const target = lo + (hi - lo) * i2 / (n - 1);
|
|
128846
|
+
let best = null;
|
|
128847
|
+
let bestD = Infinity;
|
|
128848
|
+
for (const b of asc) {
|
|
128849
|
+
if (chosen.has(b.bank))
|
|
128850
|
+
continue;
|
|
128851
|
+
const d = Math.abs(Math.log10(b.rows) - target);
|
|
128852
|
+
if (d < bestD) {
|
|
128853
|
+
bestD = d;
|
|
128854
|
+
best = b;
|
|
128855
|
+
}
|
|
128856
|
+
}
|
|
128857
|
+
if (best !== null)
|
|
128858
|
+
chosen.add(best.bank);
|
|
128859
|
+
}
|
|
128860
|
+
return nonEmpty.filter((b) => chosen.has(b.bank)).map((b) => b.bank);
|
|
128861
|
+
}
|
|
128862
|
+
const named = trimmed.split(",").map((s) => s.trim()).filter((s) => s !== "");
|
|
128863
|
+
if (named.length === 0)
|
|
128864
|
+
throw new BankSelectionError(`--banks "${spec}" selected nothing`);
|
|
128865
|
+
const known = new Set(available.map((b) => b.bank));
|
|
128866
|
+
const missing = named.filter((n) => !known.has(n));
|
|
128867
|
+
if (missing.length > 0) {
|
|
128868
|
+
throw new BankSelectionError(`--banks names ${missing.join(", ")}, which ${missing.length === 1 ? "is" : "are"} not in this instance ` + `(known: ${available.map((b) => b.bank).join(", ")})`);
|
|
128869
|
+
}
|
|
128870
|
+
return named;
|
|
128871
|
+
}
|
|
128872
|
+
function parseConcurrency(spec) {
|
|
128873
|
+
const out = new Set;
|
|
128874
|
+
for (const part of spec.split(",")) {
|
|
128875
|
+
const n = Number(part.trim());
|
|
128876
|
+
if (!Number.isInteger(n) || n < 1) {
|
|
128877
|
+
throw new BankSelectionError(`--concurrency "${spec}": "${part.trim()}" is not a positive integer`);
|
|
128878
|
+
}
|
|
128879
|
+
out.add(n);
|
|
128880
|
+
}
|
|
128881
|
+
if (out.size === 0)
|
|
128882
|
+
throw new BankSelectionError("--concurrency selected nothing");
|
|
128883
|
+
return [...out].sort((a, b) => a - b);
|
|
128884
|
+
}
|
|
128885
|
+
|
|
128886
|
+
// src/hindsight-bench/contention.ts
|
|
128887
|
+
import { spawn as spawn8 } from "node:child_process";
|
|
128888
|
+
|
|
128889
|
+
// src/hindsight-bench/db.ts
|
|
128890
|
+
import { spawnSync as spawnSync33 } from "node:child_process";
|
|
128891
|
+
var DEFAULT_CONTAINER2 = "switchroom-hindsight";
|
|
128892
|
+
var defaultRunner5 = (cmd, args, stdin) => {
|
|
128893
|
+
const r = spawnSync33(cmd, args, {
|
|
128894
|
+
stdio: "pipe",
|
|
128895
|
+
input: stdin,
|
|
128896
|
+
timeout: 120000,
|
|
128897
|
+
encoding: "utf8",
|
|
128898
|
+
maxBuffer: 16 * 1024 * 1024
|
|
128899
|
+
});
|
|
128900
|
+
if (r.error)
|
|
128901
|
+
return { status: null, stdout: "", stderr: r.error.message };
|
|
128902
|
+
return { status: r.status, stdout: r.stdout ?? "", stderr: r.stderr ?? "" };
|
|
128903
|
+
};
|
|
128904
|
+
function bootstrapSh(readOnly) {
|
|
128905
|
+
return `
|
|
128906
|
+
set -e
|
|
128907
|
+
B="$(ls -d /home/hindsight/.pg0/installation/*/bin 2>/dev/null | head -1)"
|
|
128908
|
+
PSQL="$(command -v psql 2>/dev/null || echo "\${B:-/nonexistent}/psql")"
|
|
128909
|
+
[ -x "$PSQL" ] || { echo "hindsight-bench: no psql in container" >&2; exit 3; }
|
|
128910
|
+
D=/home/hindsight/.pg0/instances/hindsight/instance.json
|
|
128911
|
+
[ -r "$D" ] || { echo "hindsight-bench: no pg0 instance descriptor" >&2; exit 3; }
|
|
128912
|
+
eval "$(python3 -c 'import json,shlex,sys
|
|
128913
|
+
d=json.load(open(sys.argv[1]))
|
|
128914
|
+
q=lambda k,dflt: shlex.quote(str(d.get(k) or dflt))
|
|
128915
|
+
print("U=%s DB=%s P=%s PW=%s"%(q("username","hindsight"),q("database","hindsight"),q("port",5432),q("password","")))' "$D")"
|
|
128916
|
+
[ -n "$PW" ] || { echo "hindsight-bench: pg0 descriptor carries no password" >&2; exit 3; }
|
|
128917
|
+
PGPASSWORD="$PW" PGOPTIONS='-c default_transaction_read_only=${readOnly ? "on" : "off"}' \\
|
|
128918
|
+
"$PSQL" -U "$U" -h /tmp -p "$P" -d "$DB" -v ON_ERROR_STOP=1 -tAF'|' -f -
|
|
128919
|
+
`;
|
|
128920
|
+
}
|
|
128921
|
+
|
|
128922
|
+
class SqlError extends Error {
|
|
128923
|
+
constructor(message) {
|
|
128924
|
+
super(message);
|
|
128925
|
+
this.name = "SqlError";
|
|
128926
|
+
}
|
|
128927
|
+
}
|
|
128928
|
+
function sql(script, opts = {}) {
|
|
128929
|
+
const run = opts.run ?? defaultRunner5;
|
|
128930
|
+
const container = opts.container ?? DEFAULT_CONTAINER2;
|
|
128931
|
+
const r = run("docker", ["exec", "-i", container, "sh", "-c", bootstrapSh(opts.writable !== true)], script);
|
|
128932
|
+
if (r.status !== 0) {
|
|
128933
|
+
throw new SqlError(`psql in ${container} exited ${r.status}: ${r.stderr.trim() || "no stderr"}`);
|
|
128934
|
+
}
|
|
128935
|
+
return r.stdout.split(`
|
|
128936
|
+
`).map((l) => l.trim()).filter((l) => l !== "");
|
|
128937
|
+
}
|
|
128938
|
+
function assertReadOnlyOrWritesAllowed(allowWrites, opts = {}) {
|
|
128939
|
+
const rows = sql(`DO $$
|
|
128940
|
+
BEGIN
|
|
128941
|
+
CREATE TEMP TABLE hindsight_bench_ro_probe(x int);
|
|
128942
|
+
EXCEPTION WHEN others THEN
|
|
128943
|
+
NULL;
|
|
128944
|
+
END $$;
|
|
128945
|
+
SELECT current_setting('transaction_read_only') || '|' ||
|
|
128946
|
+
(to_regclass('pg_temp.hindsight_bench_ro_probe') IS NOT NULL)::text;`, opts);
|
|
128947
|
+
const [declared, landed] = (rows[rows.length - 1] ?? "").split("|");
|
|
128948
|
+
const declaredReadOnly = declared === "on";
|
|
128949
|
+
const writeLanded = landed === "t";
|
|
128950
|
+
const writable = !declaredReadOnly || writeLanded;
|
|
128951
|
+
if (writable && !allowWrites) {
|
|
128952
|
+
throw new SqlError("refusing to run: the harness's database session is WRITABLE " + `(transaction_read_only=${declared ?? "?"}, temp-table write ${writeLanded ? "succeeded" : "failed"}). ` + "This harness measures a read path and must not be able to mutate a bank. " + "Pass --allow-writes only if you genuinely intend a writable session " + "(contention profile `write` needs one for its own scratch table).");
|
|
128953
|
+
}
|
|
128954
|
+
}
|
|
128955
|
+
function resetStats(opts = {}) {
|
|
128956
|
+
sql("SELECT pg_stat_reset();", { ...opts, writable: true });
|
|
128957
|
+
}
|
|
128958
|
+
function readInstanceState(opts = {}) {
|
|
128959
|
+
const run = opts.run ?? defaultRunner5;
|
|
128960
|
+
const container = opts.container ?? DEFAULT_CONTAINER2;
|
|
128961
|
+
const one = (args) => {
|
|
128962
|
+
const r = run("docker", args);
|
|
128963
|
+
if (r.status !== 0)
|
|
128964
|
+
return null;
|
|
128965
|
+
const v = r.stdout.trim();
|
|
128966
|
+
return v === "" ? null : v;
|
|
128967
|
+
};
|
|
128968
|
+
const imageTag = one(["inspect", "--format", "{{.Config.Image}}", container]);
|
|
128969
|
+
const cand = one(["exec", container, "printenv", "HINDSIGHT_API_RERANKER_MAX_CANDIDATES"]);
|
|
128970
|
+
const candNum = cand === null ? NaN : Number(cand);
|
|
128971
|
+
return { imageTag, rerankerMaxCandidates: Number.isFinite(candNum) ? candNum : null };
|
|
128972
|
+
}
|
|
128973
|
+
function num3(v) {
|
|
128974
|
+
const n = Number(v);
|
|
128975
|
+
return Number.isFinite(n) ? n : NaN;
|
|
128976
|
+
}
|
|
128977
|
+
function readDbState(opts = {}) {
|
|
128978
|
+
const settings = sql(`SELECT
|
|
128979
|
+
(SELECT setting::bigint * 8192 FROM pg_settings WHERE name = 'shared_buffers'),
|
|
128980
|
+
(SELECT setting::bigint * 8192 FROM pg_settings WHERE name = 'effective_cache_size'),
|
|
128981
|
+
(SELECT setting FROM pg_settings WHERE name = 'hnsw.ef_search'),
|
|
128982
|
+
pg_total_relation_size('memory_units'),
|
|
128983
|
+
pg_table_size('memory_units'),
|
|
128984
|
+
pg_indexes_size('memory_units'),
|
|
128985
|
+
(SELECT stats_reset::text FROM pg_stat_database WHERE datname = current_database()),
|
|
128986
|
+
(SELECT CASE WHEN heap_blks_hit + heap_blks_read = 0 THEN NULL
|
|
128987
|
+
ELSE heap_blks_hit::float8 / (heap_blks_hit + heap_blks_read) END
|
|
128988
|
+
FROM pg_statio_user_tables WHERE relname = 'memory_units'),
|
|
128989
|
+
version();`, opts);
|
|
128990
|
+
const f = (settings[0] ?? "").split("|");
|
|
128991
|
+
if (f.length < 9) {
|
|
128992
|
+
throw new SqlError(`unexpected settings row shape: ${JSON.stringify(settings[0] ?? "")}`);
|
|
128993
|
+
}
|
|
128994
|
+
const bankRows = sql(`SELECT bank_id, count(*) FROM memory_units GROUP BY bank_id ORDER BY 2 DESC;`, opts).map((l) => {
|
|
128995
|
+
const [bank, rows] = l.split("|");
|
|
128996
|
+
return { bank: bank ?? "", rows: num3(rows) };
|
|
128997
|
+
}).filter((b) => b.bank !== "" && Number.isFinite(b.rows));
|
|
128998
|
+
const largestIndexes = sql(`SELECT c.relname, pg_relation_size(c.oid), coalesce(s.idx_scan, 0)
|
|
128999
|
+
FROM pg_index i
|
|
129000
|
+
JOIN pg_class c ON c.oid = i.indexrelid
|
|
129001
|
+
LEFT JOIN pg_stat_user_indexes s ON s.indexrelid = c.oid
|
|
129002
|
+
WHERE i.indrelid = 'memory_units'::regclass
|
|
129003
|
+
ORDER BY pg_relation_size(c.oid) DESC
|
|
129004
|
+
LIMIT 8;`, opts).map((l) => {
|
|
129005
|
+
const [name, bytes, scans] = l.split("|");
|
|
129006
|
+
return { name: name ?? "", bytes: num3(bytes), scans: num3(scans) };
|
|
129007
|
+
}).filter((x) => x.name !== "");
|
|
129008
|
+
const efRaw = f[2] ?? "";
|
|
129009
|
+
return {
|
|
129010
|
+
sharedBuffersBytes: num3(f[0]),
|
|
129011
|
+
effectiveCacheSizeBytes: num3(f[1]),
|
|
129012
|
+
hnswEfSearch: efRaw === "" ? null : num3(efRaw),
|
|
129013
|
+
memoryUnitsTotalBytes: num3(f[3]),
|
|
129014
|
+
memoryUnitsHeapBytes: num3(f[4]),
|
|
129015
|
+
memoryUnitsIndexBytes: num3(f[5]),
|
|
129016
|
+
bankRows,
|
|
129017
|
+
largestIndexes,
|
|
129018
|
+
statsResetAt: (f[6] ?? "") === "" ? null : f[6],
|
|
129019
|
+
heapHitRatio: (f[7] ?? "") === "" ? null : num3(f[7]),
|
|
129020
|
+
serverVersion: f.slice(8).join("|")
|
|
129021
|
+
};
|
|
129022
|
+
}
|
|
129023
|
+
|
|
129024
|
+
// src/hindsight-bench/contention.ts
|
|
129025
|
+
var CONTENTION_APP_NAME = "hindsight-bench-contention";
|
|
129026
|
+
var SCRATCH_TABLE = "hindsight_bench_scratch";
|
|
129027
|
+
var DEFAULT_CONTENTION = {
|
|
129028
|
+
workers: 2,
|
|
129029
|
+
scanPct: 2,
|
|
129030
|
+
maxSeconds: 900
|
|
129031
|
+
};
|
|
129032
|
+
function clampInt(v, lo, hi, dflt) {
|
|
129033
|
+
const n = Math.floor(Number(v));
|
|
129034
|
+
if (!Number.isFinite(n))
|
|
129035
|
+
return dflt;
|
|
129036
|
+
return Math.min(hi, Math.max(lo, n));
|
|
129037
|
+
}
|
|
129038
|
+
function clampFloat(v, lo, hi, dflt) {
|
|
129039
|
+
const n = Number(v);
|
|
129040
|
+
if (!Number.isFinite(n))
|
|
129041
|
+
return dflt;
|
|
129042
|
+
return Math.min(hi, Math.max(lo, n));
|
|
129043
|
+
}
|
|
129044
|
+
function contentionSql(profile, scanPct, maxSeconds) {
|
|
129045
|
+
const pct2 = clampFloat(scanPct, 0.1, 100, DEFAULT_CONTENTION.scanPct);
|
|
129046
|
+
const secs = clampInt(maxSeconds, 1, 3600, DEFAULT_CONTENTION.maxSeconds);
|
|
129047
|
+
const churn = `PERFORM sum(length(m.text)) FROM memory_units m TABLESAMPLE SYSTEM (${pct2});`;
|
|
129048
|
+
const writes = profile === "write" ? `
|
|
129049
|
+
INSERT INTO ${SCRATCH_TABLE} (payload)
|
|
129050
|
+
SELECT repeat('x', 2048) FROM generate_series(1, 2000);
|
|
129051
|
+
UPDATE ${SCRATCH_TABLE} SET payload = payload || 'y' WHERE id % 3 = 0;
|
|
129052
|
+
DELETE FROM ${SCRATCH_TABLE}
|
|
129053
|
+
WHERE id < (SELECT coalesce(max(id), 0) - 20000 FROM ${SCRATCH_TABLE});` : "";
|
|
129054
|
+
return `
|
|
129055
|
+
DO $$
|
|
129056
|
+
DECLARE deadline timestamptz := clock_timestamp() + interval '${secs} seconds';
|
|
129057
|
+
BEGIN
|
|
129058
|
+
WHILE clock_timestamp() < deadline LOOP
|
|
129059
|
+
${churn}${writes}
|
|
129060
|
+
END LOOP;
|
|
129061
|
+
END $$;
|
|
129062
|
+
`;
|
|
129063
|
+
}
|
|
129064
|
+
function noContention() {
|
|
129065
|
+
return { profile: "off", workers: 0, liveBackends: 0, stop: () => {} };
|
|
129066
|
+
}
|
|
129067
|
+
function countContentionBackends(opts = {}) {
|
|
129068
|
+
const rows = sql(`SELECT count(*) FROM pg_stat_activity WHERE application_name = '${CONTENTION_APP_NAME}';`, opts);
|
|
129069
|
+
const n = Number(rows[0]);
|
|
129070
|
+
return Number.isFinite(n) ? n : 0;
|
|
129071
|
+
}
|
|
129072
|
+
function workerSh(readOnly) {
|
|
129073
|
+
return `
|
|
129074
|
+
set -e
|
|
129075
|
+
B="$(ls -d /home/hindsight/.pg0/installation/*/bin 2>/dev/null | head -1)"
|
|
129076
|
+
PSQL="$(command -v psql 2>/dev/null || echo "\${B:-/nonexistent}/psql")"
|
|
129077
|
+
[ -x "$PSQL" ] || exit 3
|
|
129078
|
+
D=/home/hindsight/.pg0/instances/hindsight/instance.json
|
|
129079
|
+
[ -r "$D" ] || exit 3
|
|
129080
|
+
eval "$(python3 -c 'import json,shlex,sys
|
|
129081
|
+
d=json.load(open(sys.argv[1]))
|
|
129082
|
+
q=lambda k,dflt: shlex.quote(str(d.get(k) or dflt))
|
|
129083
|
+
print("U=%s DB=%s P=%s PW=%s"%(q("username","hindsight"),q("database","hindsight"),q("port",5432),q("password","")))' "$D")"
|
|
129084
|
+
PGPASSWORD="$PW" PGAPPNAME='${CONTENTION_APP_NAME}' \\
|
|
129085
|
+
PGOPTIONS='-c default_transaction_read_only=${readOnly ? "on" : "off"}' \\
|
|
129086
|
+
"$PSQL" -U "$U" -h /tmp -p "$P" -d "$DB" -q -v ON_ERROR_STOP=1 -f - >/dev/null
|
|
129087
|
+
`;
|
|
129088
|
+
}
|
|
129089
|
+
|
|
129090
|
+
class ContentionError extends Error {
|
|
129091
|
+
}
|
|
129092
|
+
var sleep4 = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
129093
|
+
async function startContention(opts) {
|
|
129094
|
+
if (opts.profile === "off")
|
|
129095
|
+
return noContention();
|
|
129096
|
+
const container = opts.container ?? DEFAULT_CONTAINER2;
|
|
129097
|
+
const sqlOpts = { container, run: opts.run };
|
|
129098
|
+
const workers = clampInt(opts.workers, 1, 64, DEFAULT_CONTENTION.workers);
|
|
129099
|
+
const readOnly = opts.profile !== "write";
|
|
129100
|
+
if (opts.profile === "write") {
|
|
129101
|
+
sql(`CREATE TABLE IF NOT EXISTS ${SCRATCH_TABLE} (id bigserial PRIMARY KEY, payload text);
|
|
129102
|
+
TRUNCATE ${SCRATCH_TABLE};`, { ...sqlOpts, writable: true });
|
|
129103
|
+
}
|
|
129104
|
+
const script = contentionSql(opts.profile, opts.scanPct, opts.maxSeconds);
|
|
129105
|
+
const children = [];
|
|
129106
|
+
let workerStderr = "";
|
|
129107
|
+
for (let i2 = 0;i2 < workers; i2++) {
|
|
129108
|
+
const child = spawn8("docker", ["exec", "-i", container, "sh", "-c", workerSh(readOnly)], {
|
|
129109
|
+
stdio: ["pipe", "ignore", "pipe"],
|
|
129110
|
+
detached: false
|
|
129111
|
+
});
|
|
129112
|
+
child.stdin?.end(script);
|
|
129113
|
+
child.stderr?.on("data", (d) => {
|
|
129114
|
+
if (workerStderr.length < 4000)
|
|
129115
|
+
workerStderr += d.toString();
|
|
129116
|
+
});
|
|
129117
|
+
child.on("error", (e) => {
|
|
129118
|
+
workerStderr += `${e.message}
|
|
129119
|
+
`;
|
|
129120
|
+
});
|
|
129121
|
+
children.push(child);
|
|
129122
|
+
}
|
|
129123
|
+
let stopped = false;
|
|
129124
|
+
const killAll = () => {
|
|
129125
|
+
for (const c of children) {
|
|
129126
|
+
try {
|
|
129127
|
+
c.kill("SIGKILL");
|
|
129128
|
+
} catch {}
|
|
129129
|
+
}
|
|
129130
|
+
};
|
|
129131
|
+
let liveBackends = 0;
|
|
129132
|
+
for (let attempt = 0;attempt < 10 && liveBackends === 0; attempt++) {
|
|
129133
|
+
await sleep4(500);
|
|
129134
|
+
liveBackends = countContentionBackends(sqlOpts);
|
|
129135
|
+
}
|
|
129136
|
+
if (liveBackends === 0) {
|
|
129137
|
+
killAll();
|
|
129138
|
+
throw new ContentionError(`contention profile "${opts.profile}" started ${workers} worker(s) but none attached to ` + `PostgreSQL within 5s \u2014 the measurement would be of an IDLE system. ` + `worker stderr: ${workerStderr.trim() || "(none)"}`);
|
|
129139
|
+
}
|
|
129140
|
+
return {
|
|
129141
|
+
profile: opts.profile,
|
|
129142
|
+
workers,
|
|
129143
|
+
liveBackends,
|
|
129144
|
+
stop: () => {
|
|
129145
|
+
if (stopped)
|
|
129146
|
+
return;
|
|
129147
|
+
stopped = true;
|
|
129148
|
+
killAll();
|
|
129149
|
+
try {
|
|
129150
|
+
sql(`SELECT pg_terminate_backend(pid) FROM pg_stat_activity
|
|
129151
|
+
WHERE application_name = '${CONTENTION_APP_NAME}' AND pid <> pg_backend_pid();`, sqlOpts);
|
|
129152
|
+
} catch {}
|
|
129153
|
+
if (opts.profile === "write") {
|
|
129154
|
+
try {
|
|
129155
|
+
sql(`DROP TABLE IF EXISTS ${SCRATCH_TABLE};`, { ...sqlOpts, writable: true });
|
|
129156
|
+
} catch {}
|
|
129157
|
+
}
|
|
129158
|
+
}
|
|
129159
|
+
};
|
|
129160
|
+
}
|
|
129161
|
+
|
|
129162
|
+
// src/hindsight-bench/plot.ts
|
|
129163
|
+
var W2 = 900;
|
|
129164
|
+
var H = 520;
|
|
129165
|
+
var PAD = { top: 46, right: 210, bottom: 56, left: 76 };
|
|
129166
|
+
var COLOURS = ["#0072B2", "#D55E00", "#009E73", "#CC79A7", "#E69F00", "#56B4E9", "#F0E442", "#000000"];
|
|
129167
|
+
function escapeXml(s) {
|
|
129168
|
+
return s.replace(/[<>&"']/g, (c) => `&#${c.charCodeAt(0)};`);
|
|
129169
|
+
}
|
|
129170
|
+
function buildSeries(results) {
|
|
129171
|
+
const series = [];
|
|
129172
|
+
for (const r of results) {
|
|
129173
|
+
const byConc = new Map;
|
|
129174
|
+
for (const c of r.cells) {
|
|
129175
|
+
if (!Number.isFinite(c.stats.p95) || c.rows <= 0)
|
|
129176
|
+
continue;
|
|
129177
|
+
const arr = byConc.get(c.concurrency) ?? [];
|
|
129178
|
+
arr.push({ x: c.rows, y: c.stats.p95 });
|
|
129179
|
+
byConc.set(c.concurrency, arr);
|
|
129180
|
+
}
|
|
129181
|
+
const tag = r.config.label || r.config.startedAt;
|
|
129182
|
+
for (const conc of [...byConc.keys()].sort((a, b) => a - b)) {
|
|
129183
|
+
const pts = byConc.get(conc).sort((a, b) => a.x - b.x);
|
|
129184
|
+
series.push({ label: `${tag} \u00b7 c=${conc}`, points: pts });
|
|
129185
|
+
}
|
|
129186
|
+
}
|
|
129187
|
+
return series;
|
|
129188
|
+
}
|
|
129189
|
+
function renderPlot(results, title = "Hindsight recall p95 vs bank size") {
|
|
129190
|
+
const series = buildSeries(results);
|
|
129191
|
+
const all = series.flatMap((s) => s.points);
|
|
129192
|
+
if (all.length === 0) {
|
|
129193
|
+
return `<svg xmlns="http://www.w3.org/2000/svg" width="${W2}" height="80"><text x="16" y="44" font-family="sans-serif" font-size="14">no plottable cells (every p95 was non-finite)</text></svg>`;
|
|
129194
|
+
}
|
|
129195
|
+
const xs = all.map((p) => Math.log10(p.x));
|
|
129196
|
+
const ys = all.map((p) => p.y);
|
|
129197
|
+
const x0 = Math.min(...xs);
|
|
129198
|
+
const x1 = Math.max(...xs);
|
|
129199
|
+
const y1 = Math.max(...ys);
|
|
129200
|
+
const xSpan = x1 - x0 === 0 ? 1 : x1 - x0;
|
|
129201
|
+
const ySpan = y1 === 0 ? 1 : y1 * 1.08;
|
|
129202
|
+
const px = (v) => PAD.left + (Math.log10(v) - x0) / xSpan * (W2 - PAD.left - PAD.right);
|
|
129203
|
+
const py = (v) => H - PAD.bottom - v / ySpan * (H - PAD.top - PAD.bottom);
|
|
129204
|
+
const parts = [];
|
|
129205
|
+
parts.push(`<svg xmlns="http://www.w3.org/2000/svg" width="${W2}" height="${H}" viewBox="0 0 ${W2} ${H}">`);
|
|
129206
|
+
parts.push(`<rect width="${W2}" height="${H}" fill="#ffffff"/>`);
|
|
129207
|
+
parts.push(`<text x="${PAD.left}" y="26" font-family="sans-serif" font-size="16" font-weight="600">${escapeXml(title)}</text>`);
|
|
129208
|
+
for (let i2 = 0;i2 <= 4; i2++) {
|
|
129209
|
+
const v = ySpan / 4 * i2;
|
|
129210
|
+
const y = py(v);
|
|
129211
|
+
parts.push(`<line x1="${PAD.left}" y1="${y}" x2="${W2 - PAD.right}" y2="${y}" stroke="#e6e6e6"/>`);
|
|
129212
|
+
parts.push(`<text x="${PAD.left - 8}" y="${y + 4}" text-anchor="end" font-family="sans-serif" font-size="11" fill="#555">${Math.round(v)}</text>`);
|
|
129213
|
+
}
|
|
129214
|
+
for (let d = Math.floor(x0);d <= Math.ceil(x1); d++) {
|
|
129215
|
+
const v = 10 ** d;
|
|
129216
|
+
if (Math.log10(v) < x0 || Math.log10(v) > x1)
|
|
129217
|
+
continue;
|
|
129218
|
+
const x = px(v);
|
|
129219
|
+
parts.push(`<line x1="${x}" y1="${PAD.top}" x2="${x}" y2="${H - PAD.bottom}" stroke="#f0f0f0"/>`);
|
|
129220
|
+
parts.push(`<text x="${x}" y="${H - PAD.bottom + 18}" text-anchor="middle" font-family="sans-serif" font-size="11" fill="#555">10^${d}</text>`);
|
|
129221
|
+
}
|
|
129222
|
+
parts.push(`<line x1="${PAD.left}" y1="${H - PAD.bottom}" x2="${W2 - PAD.right}" y2="${H - PAD.bottom}" stroke="#333"/>`);
|
|
129223
|
+
parts.push(`<line x1="${PAD.left}" y1="${PAD.top}" x2="${PAD.left}" y2="${H - PAD.bottom}" stroke="#333"/>`);
|
|
129224
|
+
parts.push(`<text x="${(PAD.left + W2 - PAD.right) / 2}" y="${H - 14}" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#333">memory_units rows in bank (log scale)</text>`);
|
|
129225
|
+
parts.push(`<text x="18" y="${(PAD.top + H - PAD.bottom) / 2}" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#333" transform="rotate(-90 18 ${(PAD.top + H - PAD.bottom) / 2})">recall p95 (ms)</text>`);
|
|
129226
|
+
series.forEach((s, i2) => {
|
|
129227
|
+
const colour = COLOURS[i2 % COLOURS.length];
|
|
129228
|
+
const d = s.points.map((p, j) => `${j === 0 ? "M" : "L"}${px(p.x).toFixed(1)},${py(p.y).toFixed(1)}`).join(" ");
|
|
129229
|
+
parts.push(`<path d="${d}" fill="none" stroke="${colour}" stroke-width="2"/>`);
|
|
129230
|
+
for (const p of s.points) {
|
|
129231
|
+
parts.push(`<circle cx="${px(p.x).toFixed(1)}" cy="${py(p.y).toFixed(1)}" r="3" fill="${colour}"/>`);
|
|
129232
|
+
}
|
|
129233
|
+
const ly = PAD.top + 6 + i2 * 18;
|
|
129234
|
+
parts.push(`<line x1="${W2 - PAD.right + 12}" y1="${ly}" x2="${W2 - PAD.right + 32}" y2="${ly}" stroke="${colour}" stroke-width="2"/>`);
|
|
129235
|
+
parts.push(`<text x="${W2 - PAD.right + 38}" y="${ly + 4}" font-family="sans-serif" font-size="11" fill="#333">${escapeXml(s.label)}</text>`);
|
|
129236
|
+
});
|
|
129237
|
+
parts.push("</svg>");
|
|
129238
|
+
return parts.join(`
|
|
129239
|
+
`);
|
|
129240
|
+
}
|
|
129241
|
+
|
|
129242
|
+
// src/hindsight-bench/recall.ts
|
|
129243
|
+
var QUERY_SET_ID = "generic-v1";
|
|
129244
|
+
var RECALL_QUERIES = [
|
|
129245
|
+
"what did we decide about the deployment process",
|
|
129246
|
+
"open issues that are still blocked",
|
|
129247
|
+
"postgres configuration and tuning decisions",
|
|
129248
|
+
"how does authentication work here",
|
|
129249
|
+
"what went wrong in the last incident",
|
|
129250
|
+
"preferences about how to write reports",
|
|
129251
|
+
"recent changes to the build pipeline",
|
|
129252
|
+
"who is responsible for the review process",
|
|
129253
|
+
"outstanding follow-up work and commitments",
|
|
129254
|
+
"container restart and rollout procedure",
|
|
129255
|
+
"why was that approach rejected",
|
|
129256
|
+
"measurement results and benchmark numbers",
|
|
129257
|
+
"scheduling and cron behaviour",
|
|
129258
|
+
"known limitations of the current design",
|
|
129259
|
+
"what changed in the most recent release",
|
|
129260
|
+
"error handling and retry behaviour",
|
|
129261
|
+
"naming conventions used in this project",
|
|
129262
|
+
"cost and quota considerations",
|
|
129263
|
+
"notes about testing and coverage",
|
|
129264
|
+
"security constraints that must not be crossed"
|
|
129265
|
+
];
|
|
129266
|
+
function restBase(url) {
|
|
129267
|
+
return url.replace(/\/mcp\/?$/, "").replace(/\/$/, "");
|
|
129268
|
+
}
|
|
129269
|
+
async function recallOnce(bank, query, opts) {
|
|
129270
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
129271
|
+
const url = `${restBase(opts.apiUrl)}/v1/default/banks/${encodeURIComponent(bank)}/memories/recall`;
|
|
129272
|
+
const body = JSON.stringify({
|
|
129273
|
+
query,
|
|
129274
|
+
types: ["world", "experience"],
|
|
129275
|
+
budget: opts.budget,
|
|
129276
|
+
max_tokens: opts.maxTokens,
|
|
129277
|
+
trace: opts.trace === true
|
|
129278
|
+
});
|
|
129279
|
+
const started = performance.now();
|
|
129280
|
+
try {
|
|
129281
|
+
const res = await fetchImpl(url, {
|
|
129282
|
+
method: "POST",
|
|
129283
|
+
headers: { "content-type": "application/json" },
|
|
129284
|
+
body,
|
|
129285
|
+
redirect: "error",
|
|
129286
|
+
signal: AbortSignal.timeout(opts.timeoutMs)
|
|
129287
|
+
});
|
|
129288
|
+
const text = await res.text();
|
|
129289
|
+
const ms = performance.now() - started;
|
|
129290
|
+
if (!res.ok)
|
|
129291
|
+
return { ok: false, ms, results: 0, error: `HTTP ${res.status}` };
|
|
129292
|
+
let parsed;
|
|
129293
|
+
try {
|
|
129294
|
+
parsed = JSON.parse(text);
|
|
129295
|
+
} catch {
|
|
129296
|
+
return { ok: false, ms, results: 0, error: "unparseable JSON body" };
|
|
129297
|
+
}
|
|
129298
|
+
const results = Array.isArray(parsed.results) ? parsed.results.length : 0;
|
|
129299
|
+
return opts.trace === true ? { ok: true, ms, results, trace: parsed.trace } : { ok: true, ms, results };
|
|
129300
|
+
} catch (e) {
|
|
129301
|
+
return {
|
|
129302
|
+
ok: false,
|
|
129303
|
+
ms: performance.now() - started,
|
|
129304
|
+
results: 0,
|
|
129305
|
+
error: e.name === "TimeoutError" ? `timeout >${opts.timeoutMs}ms` : e.message
|
|
129306
|
+
};
|
|
129307
|
+
}
|
|
129308
|
+
}
|
|
129309
|
+
|
|
129310
|
+
// src/hindsight-bench/stats.ts
|
|
129311
|
+
function percentile2(values, p) {
|
|
129312
|
+
if (values.length === 0)
|
|
129313
|
+
return NaN;
|
|
129314
|
+
if (!(p > 0) || p > 1)
|
|
129315
|
+
throw new RangeError(`percentile p must be in (0, 1], got ${p}`);
|
|
129316
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
129317
|
+
const rank = Math.min(sorted.length, Math.max(1, Math.ceil(p * sorted.length)));
|
|
129318
|
+
return sorted[rank - 1];
|
|
129319
|
+
}
|
|
129320
|
+
var BOOTSTRAP_REPS = 2000;
|
|
129321
|
+
function mulberry32(seed) {
|
|
129322
|
+
let a = seed >>> 0;
|
|
129323
|
+
return () => {
|
|
129324
|
+
a = a + 1831565813 >>> 0;
|
|
129325
|
+
let t = Math.imul(a ^ a >>> 15, 1 | a);
|
|
129326
|
+
t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
|
|
129327
|
+
return ((t ^ t >>> 14) >>> 0) / 4294967296;
|
|
129328
|
+
};
|
|
129329
|
+
}
|
|
129330
|
+
function bootstrapP95Ci(values, reps = BOOTSTRAP_REPS) {
|
|
129331
|
+
const n = values.length;
|
|
129332
|
+
if (n === 0)
|
|
129333
|
+
return [NaN, NaN];
|
|
129334
|
+
if (n === 1)
|
|
129335
|
+
return [values[0], values[0]];
|
|
129336
|
+
const pool = [...values].sort((a, b) => a - b);
|
|
129337
|
+
const seed = n * 2654435761 + Math.round(pool.reduce((a, b) => a + b, 0));
|
|
129338
|
+
const rnd = mulberry32(seed);
|
|
129339
|
+
const reduced = new Array(reps);
|
|
129340
|
+
const draw = new Array(n);
|
|
129341
|
+
for (let r = 0;r < reps; r++) {
|
|
129342
|
+
for (let i2 = 0;i2 < n; i2++)
|
|
129343
|
+
draw[i2] = pool[Math.floor(rnd() * n)];
|
|
129344
|
+
reduced[r] = percentile2(draw, 0.95);
|
|
129345
|
+
}
|
|
129346
|
+
reduced.sort((a, b) => a - b);
|
|
129347
|
+
const lo = reduced[Math.max(0, Math.floor(0.025 * reps) - 1)];
|
|
129348
|
+
const hi = reduced[Math.min(reps - 1, Math.ceil(0.975 * reps) - 1)];
|
|
129349
|
+
return [lo, hi];
|
|
129350
|
+
}
|
|
129351
|
+
function summarize(samplesMs, errors4) {
|
|
129352
|
+
const n = samplesMs.length;
|
|
129353
|
+
if (n === 0) {
|
|
129354
|
+
return {
|
|
129355
|
+
n: 0,
|
|
129356
|
+
errors: errors4,
|
|
129357
|
+
min: NaN,
|
|
129358
|
+
p50: NaN,
|
|
129359
|
+
p95: NaN,
|
|
129360
|
+
p99: NaN,
|
|
129361
|
+
max: NaN,
|
|
129362
|
+
mean: NaN,
|
|
129363
|
+
stddev: NaN,
|
|
129364
|
+
cv: NaN,
|
|
129365
|
+
p95CiLow: NaN,
|
|
129366
|
+
p95CiHigh: NaN
|
|
129367
|
+
};
|
|
129368
|
+
}
|
|
129369
|
+
const sorted = [...samplesMs].sort((a, b) => a - b);
|
|
129370
|
+
const sum = sorted.reduce((a, b) => a + b, 0);
|
|
129371
|
+
const mean = sum / n;
|
|
129372
|
+
const variance = sorted.reduce((a, v) => a + (v - mean) ** 2, 0) / n;
|
|
129373
|
+
const stddev = Math.sqrt(variance);
|
|
129374
|
+
const [p95CiLow, p95CiHigh] = bootstrapP95Ci(sorted);
|
|
129375
|
+
return {
|
|
129376
|
+
n,
|
|
129377
|
+
errors: errors4,
|
|
129378
|
+
min: sorted[0],
|
|
129379
|
+
p50: percentile2(sorted, 0.5),
|
|
129380
|
+
p95: percentile2(sorted, 0.95),
|
|
129381
|
+
p99: percentile2(sorted, 0.99),
|
|
129382
|
+
max: sorted[n - 1],
|
|
129383
|
+
mean,
|
|
129384
|
+
stddev,
|
|
129385
|
+
cv: mean === 0 ? 0 : stddev / mean,
|
|
129386
|
+
p95CiLow,
|
|
129387
|
+
p95CiHigh
|
|
129388
|
+
};
|
|
129389
|
+
}
|
|
129390
|
+
function round(v, dp = 1) {
|
|
129391
|
+
if (!Number.isFinite(v))
|
|
129392
|
+
return v;
|
|
129393
|
+
const f = 10 ** dp;
|
|
129394
|
+
return Math.round(v * f) / f;
|
|
129395
|
+
}
|
|
129396
|
+
|
|
129397
|
+
// src/hindsight-bench/report.ts
|
|
129398
|
+
var DEFAULT_TOLERANCE = 0.1;
|
|
129399
|
+
var cellKey = (bank, concurrency) => `${bank}@c${concurrency}`;
|
|
129400
|
+
function duplicateCellKeys(cells) {
|
|
129401
|
+
const seen = new Set;
|
|
129402
|
+
const dupes = new Set;
|
|
129403
|
+
for (const c of cells) {
|
|
129404
|
+
const key = cellKey(c.bank, c.concurrency);
|
|
129405
|
+
if (seen.has(key))
|
|
129406
|
+
dupes.add(key);
|
|
129407
|
+
seen.add(key);
|
|
129408
|
+
}
|
|
129409
|
+
return [...dupes];
|
|
129410
|
+
}
|
|
129411
|
+
var MAX_CELL_ERROR_RATE = 0.1;
|
|
129412
|
+
function ungradeable(r) {
|
|
129413
|
+
const out = [];
|
|
129414
|
+
for (const c of r.cells) {
|
|
129415
|
+
const { n, errors: errors4 } = c.stats;
|
|
129416
|
+
const key = cellKey(c.bank, c.concurrency);
|
|
129417
|
+
if (n === 0 || !Number.isFinite(c.stats.p95)) {
|
|
129418
|
+
out.push({ key, label: `${key} (no-samples, ${errors4} errors)` });
|
|
129419
|
+
} else if (errors4 / (n + errors4) > MAX_CELL_ERROR_RATE) {
|
|
129420
|
+
out.push({ key, label: `${key} (error-rate ${round(errors4 / (n + errors4) * 100, 0)}%, n=${n})` });
|
|
129421
|
+
}
|
|
129422
|
+
}
|
|
129423
|
+
return out;
|
|
129424
|
+
}
|
|
129425
|
+
function compareRuns(a, b, tolerance = DEFAULT_TOLERANCE) {
|
|
129426
|
+
const bByKey = new Map(b.cells.map((c) => [cellKey(c.bank, c.concurrency), c]));
|
|
129427
|
+
const seen = new Set;
|
|
129428
|
+
const cells = [];
|
|
129429
|
+
const unmatched = [];
|
|
129430
|
+
const bad = [...ungradeable(a), ...ungradeable(b)];
|
|
129431
|
+
const badKeys = new Set(bad.map((u) => u.key));
|
|
129432
|
+
for (const ca of a.cells) {
|
|
129433
|
+
const key = cellKey(ca.bank, ca.concurrency);
|
|
129434
|
+
seen.add(key);
|
|
129435
|
+
const cb = bByKey.get(key);
|
|
129436
|
+
if (badKeys.has(key))
|
|
129437
|
+
continue;
|
|
129438
|
+
if (cb === undefined || !Number.isFinite(ca.stats.p95) || !Number.isFinite(cb.stats.p95) || ca.stats.p95 === 0) {
|
|
129439
|
+
unmatched.push(key);
|
|
129440
|
+
continue;
|
|
129441
|
+
}
|
|
129442
|
+
const relDelta = Math.abs(cb.stats.p95 - ca.stats.p95) / ca.stats.p95;
|
|
129443
|
+
const ciLow = ca.stats.p95CiLow;
|
|
129444
|
+
const ciHigh = ca.stats.p95CiHigh;
|
|
129445
|
+
const noiseFloor = Number.isFinite(ciLow) && Number.isFinite(ciHigh) ? (ciHigh - ciLow) / 2 / ca.stats.p95 : NaN;
|
|
129446
|
+
cells.push({
|
|
129447
|
+
bank: ca.bank,
|
|
129448
|
+
concurrency: ca.concurrency,
|
|
129449
|
+
p95A: ca.stats.p95,
|
|
129450
|
+
p95B: cb.stats.p95,
|
|
129451
|
+
relDelta,
|
|
129452
|
+
within: relDelta <= tolerance,
|
|
129453
|
+
noiseFloor,
|
|
129454
|
+
explainedByNoise: Number.isFinite(noiseFloor) && relDelta <= noiseFloor
|
|
129455
|
+
});
|
|
129456
|
+
}
|
|
129457
|
+
for (const key of bByKey.keys())
|
|
129458
|
+
if (!seen.has(key) && !badKeys.has(key))
|
|
129459
|
+
unmatched.push(key);
|
|
129460
|
+
const worstRelDelta = cells.reduce((m, c) => Math.max(m, c.relDelta), 0);
|
|
129461
|
+
const ungradeableLabels = [...new Set(bad.map((u) => u.label))];
|
|
129462
|
+
return {
|
|
129463
|
+
tolerance,
|
|
129464
|
+
cells,
|
|
129465
|
+
unmatched,
|
|
129466
|
+
ungradeable: ungradeableLabels,
|
|
129467
|
+
worstRelDelta,
|
|
129468
|
+
pass: unmatched.length === 0 && ungradeableLabels.length === 0 && cells.length > 0 && cells.every((c) => c.within)
|
|
129469
|
+
};
|
|
129470
|
+
}
|
|
129471
|
+
var CONTENTION_MIN_REL_DELTA = 0.1;
|
|
129472
|
+
function compareContention(idle, loaded, minRelDelta = CONTENTION_MIN_REL_DELTA) {
|
|
129473
|
+
const loadedByKey = new Map(loaded.cells.map((c) => [cellKey(c.bank, c.concurrency), c]));
|
|
129474
|
+
const cells = [];
|
|
129475
|
+
const bad = [...ungradeable(idle), ...ungradeable(loaded)];
|
|
129476
|
+
const badKeys = new Set(bad.map((u) => u.key));
|
|
129477
|
+
for (const ci of idle.cells) {
|
|
129478
|
+
if (badKeys.has(cellKey(ci.bank, ci.concurrency)))
|
|
129479
|
+
continue;
|
|
129480
|
+
const cl = loadedByKey.get(cellKey(ci.bank, ci.concurrency));
|
|
129481
|
+
if (cl === undefined || !Number.isFinite(ci.stats.p95) || ci.stats.p95 === 0 || !Number.isFinite(cl.stats.p95)) {
|
|
129482
|
+
continue;
|
|
129483
|
+
}
|
|
129484
|
+
cells.push({
|
|
129485
|
+
bank: ci.bank,
|
|
129486
|
+
concurrency: ci.concurrency,
|
|
129487
|
+
idleP95: ci.stats.p95,
|
|
129488
|
+
loadedP95: cl.stats.p95,
|
|
129489
|
+
relDelta: (cl.stats.p95 - ci.stats.p95) / ci.stats.p95
|
|
129490
|
+
});
|
|
129491
|
+
}
|
|
129492
|
+
const raised = cells.filter((c) => c.relDelta > 0).length;
|
|
129493
|
+
const sorted = cells.map((c) => c.relDelta).sort((a, b) => a - b);
|
|
129494
|
+
const medianRelDelta = sorted.length === 0 ? 0 : sorted[Math.floor((sorted.length - 1) / 2)];
|
|
129495
|
+
const ungradeableLabels = [...new Set(bad.map((u) => u.label))];
|
|
129496
|
+
return {
|
|
129497
|
+
cells,
|
|
129498
|
+
raised,
|
|
129499
|
+
medianRelDelta,
|
|
129500
|
+
minRelDelta,
|
|
129501
|
+
ungradeable: ungradeableLabels,
|
|
129502
|
+
pass: cells.length > 0 && ungradeableLabels.length === 0 && raised > cells.length / 2 && medianRelDelta >= minRelDelta
|
|
129503
|
+
};
|
|
129504
|
+
}
|
|
129505
|
+
var mb = (bytes) => Number.isFinite(bytes) ? `${Math.round(bytes / 1024 / 1024)} MB` : "?";
|
|
129506
|
+
function pad5(s, n) {
|
|
129507
|
+
return s.length >= n ? s : s + " ".repeat(n - s.length);
|
|
129508
|
+
}
|
|
129509
|
+
function padL(s, n) {
|
|
129510
|
+
return s.length >= n ? s : " ".repeat(n - s.length) + s;
|
|
129511
|
+
}
|
|
129512
|
+
function formatSummary(r) {
|
|
129513
|
+
const L = [];
|
|
129514
|
+
const c = r.config;
|
|
129515
|
+
L.push(`hindsight recall bench \u2014 ${c.label || "(unlabelled)"}`);
|
|
129516
|
+
L.push(` started ${c.startedAt} \u00b7 ${round(r.durationS, 0)}s \u00b7 schema v${r.schema}`);
|
|
129517
|
+
L.push(` banks=${c.banks.length} concurrency=${c.concurrency.join(",")} samples=${c.samples} ` + `warmup=${c.warmup} query-set=${c.querySet} budget=${c.budget}`);
|
|
129518
|
+
L.push(` contention=${c.contention}${c.contention === "off" ? "" : ` workers=${c.contentionWorkers}`}` + ` stats-reset=${c.statsReset} allow-writes=${c.allowWrites}`);
|
|
129519
|
+
L.push("");
|
|
129520
|
+
L.push(` instance image=${r.instance.imageTag ?? "unknown"} ` + `reranker_max_candidates=${r.instance.rerankerMaxCandidates ?? "unset"}`);
|
|
129521
|
+
L.push("");
|
|
129522
|
+
L.push(" db state");
|
|
129523
|
+
L.push(` shared_buffers=${mb(r.db.sharedBuffersBytes)} effective_cache_size=${mb(r.db.effectiveCacheSizeBytes)}` + ` hnsw.ef_search=${r.db.hnswEfSearch ?? "unset"}`);
|
|
129524
|
+
L.push(` memory_units total=${mb(r.db.memoryUnitsTotalBytes)} (heap ${mb(r.db.memoryUnitsHeapBytes)} + ` + `indexes ${mb(r.db.memoryUnitsIndexBytes)})`);
|
|
129525
|
+
const fit = r.db.memoryUnitsTotalBytes / r.db.sharedBuffersBytes;
|
|
129526
|
+
L.push(` working set / shared_buffers = ${round(fit * 100, 1)}%`);
|
|
129527
|
+
L.push(` stats_reset=${r.db.statsResetAt ?? "NEVER (cache-hit ratios are cumulative since initdb)"}` + ` heap_hit=${r.db.heapHitRatio === null ? "n/a" : `${round(r.db.heapHitRatio * 100, 2)}%`}`);
|
|
129528
|
+
L.push("");
|
|
129529
|
+
L.push(` ${pad5("bank", 16)}${padL("rows", 8)}${padL("conc", 6)}${padL("n", 6)}${padL("err", 5)}` + `${padL("p50", 9)}${padL("p95", 9)}${padL("p99", 9)}${padL("max", 9)}${padL("cv", 7)}${padL("hits", 7)}`);
|
|
129530
|
+
for (const cell of r.cells) {
|
|
129531
|
+
const s = cell.stats;
|
|
129532
|
+
L.push(` ${pad5(cell.bank, 16)}${padL(String(cell.rows), 8)}${padL(String(cell.concurrency), 6)}` + `${padL(String(s.n), 6)}${padL(String(s.errors), 5)}` + `${padL(Number.isFinite(s.p50) ? s.p50.toFixed(0) : "-", 9)}` + `${padL(Number.isFinite(s.p95) ? s.p95.toFixed(0) : "-", 9)}` + `${padL(Number.isFinite(s.p99) ? s.p99.toFixed(0) : "-", 9)}` + `${padL(Number.isFinite(s.max) ? s.max.toFixed(0) : "-", 9)}` + `${padL(Number.isFinite(s.cv) ? s.cv.toFixed(2) : "-", 7)}` + `${padL(cell.meanResults.toFixed(1), 7)}`);
|
|
129533
|
+
}
|
|
129534
|
+
const empties = r.cells.filter((c2) => c2.zeroResultCalls > 0);
|
|
129535
|
+
if (empties.length > 0) {
|
|
129536
|
+
L.push("");
|
|
129537
|
+
L.push(" ! cells with zero-result recalls (latency there is not measuring retrieval):");
|
|
129538
|
+
for (const c2 of empties)
|
|
129539
|
+
L.push(` ${c2.bank} c=${c2.concurrency}: ${c2.zeroResultCalls}/${c2.stats.n} empty`);
|
|
129540
|
+
}
|
|
129541
|
+
if (r.arms !== null && r.arms.length > 0) {
|
|
129542
|
+
L.push("");
|
|
129543
|
+
L.push(" arm attribution (SEPARATE traced pass \u2014 not comparable to the latency table above)");
|
|
129544
|
+
L.push(` ${pad5("bank", 16)}${pad5("method", 12)}${pad5("fact_type", 12)}${padL("n", 5)}${padL("p50", 9)}${padL("p95", 9)}`);
|
|
129545
|
+
for (const a of r.arms) {
|
|
129546
|
+
L.push(` ${pad5(a.bank, 16)}${pad5(a.method, 12)}${pad5(a.fact_type, 12)}${padL(String(a.n), 5)}` + `${padL(a.p50.toFixed(1), 9)}${padL(a.p95.toFixed(1), 9)}`);
|
|
129547
|
+
}
|
|
129548
|
+
}
|
|
129549
|
+
return L.join(`
|
|
129550
|
+
`);
|
|
129551
|
+
}
|
|
129552
|
+
function formatReproducibility(rep) {
|
|
129553
|
+
const L = [];
|
|
129554
|
+
L.push(`reproducibility (AC1) \u2014 tolerance \u00b1${round(rep.tolerance * 100, 0)}% on p95 per cell`);
|
|
129555
|
+
L.push(` ${pad5("bank", 16)}${padL("conc", 6)}${padL("p95 A", 10)}${padL("p95 B", 10)}` + `${padL("delta", 9)}${padL("\u00b1noise", 9)} verdict`);
|
|
129556
|
+
for (const c of rep.cells) {
|
|
129557
|
+
L.push(` ${pad5(c.bank, 16)}${padL(String(c.concurrency), 6)}${padL(c.p95A.toFixed(0), 10)}` + `${padL(c.p95B.toFixed(0), 10)}${padL(`${round(c.relDelta * 100, 1)}%`, 9)}` + `${padL(Number.isFinite(c.noiseFloor) ? `${round(c.noiseFloor * 100, 1)}%` : "-", 9)} ` + `${c.within ? "ok" : c.explainedByNoise ? "OUT*" : "OUT"}`);
|
|
129558
|
+
}
|
|
129559
|
+
for (const u of rep.unmatched)
|
|
129560
|
+
L.push(` ${u}: UNMATCHED (present in only one run, or no usable p95)`);
|
|
129561
|
+
for (const u of rep.ungradeable)
|
|
129562
|
+
L.push(` ${u}: UNGRADEABLE \u2014 the measurement failed here`);
|
|
129563
|
+
L.push(` worst delta ${round(rep.worstRelDelta * 100, 1)}% \u2192 ${rep.pass ? "PASS" : "FAIL"}`);
|
|
129564
|
+
const noisy = rep.cells.filter((c) => c.explainedByNoise).length;
|
|
129565
|
+
const floors = rep.cells.map((c) => c.noiseFloor).filter((v) => Number.isFinite(v));
|
|
129566
|
+
if (floors.length > 0) {
|
|
129567
|
+
const median2 = [...floors].sort((a, b) => a - b)[Math.floor(floors.length / 2)];
|
|
129568
|
+
L.push(` median \u00b1noise ${round(median2 * 100, 1)}% \u2014 the smallest delta this sample count can ` + `distinguish from luck; ${noisy}/${rep.cells.length} cells (OUT*) are inside it`);
|
|
129569
|
+
if (median2 > rep.tolerance) {
|
|
129570
|
+
L.push(` NOTE: the median noise floor EXCEEDS the \u00b1${round(rep.tolerance * 100, 0)}% gate \u2014 ` + `at this sample count the gate is not attainable regardless of how stable the ` + `system is. Raise --samples or grade a lower percentile; do not widen the tolerance to fit.`);
|
|
129571
|
+
}
|
|
129572
|
+
}
|
|
129573
|
+
return L.join(`
|
|
129574
|
+
`);
|
|
129575
|
+
}
|
|
129576
|
+
function formatContention(rep) {
|
|
129577
|
+
const L = [];
|
|
129578
|
+
L.push(`contention (AC4) \u2014 contended p95 must exceed idle p95 in a majority of cells, ` + `median movement \u2265 ${round(rep.minRelDelta * 100, 0)}%`);
|
|
129579
|
+
L.push(` ${pad5("bank", 16)}${padL("conc", 6)}${padL("idle p95", 11)}${padL("load p95", 11)}${padL("delta", 9)}`);
|
|
129580
|
+
for (const c of rep.cells) {
|
|
129581
|
+
L.push(` ${pad5(c.bank, 16)}${padL(String(c.concurrency), 6)}${padL(c.idleP95.toFixed(0), 11)}` + `${padL(c.loadedP95.toFixed(0), 11)}${padL(`${round(c.relDelta * 100, 1)}%`, 9)}`);
|
|
129582
|
+
}
|
|
129583
|
+
for (const u of rep.ungradeable)
|
|
129584
|
+
L.push(` ${u}: UNGRADEABLE \u2014 the measurement failed here`);
|
|
129585
|
+
L.push(` raised in ${rep.raised}/${rep.cells.length} cells, median ${round(rep.medianRelDelta * 100, 1)}% \u2192 ` + `${rep.pass ? "PASS" : "FAIL"}`);
|
|
129586
|
+
if (rep.ungradeable.length > 0) {
|
|
129587
|
+
L.push(` FAIL is on the MEASUREMENT, not the generator: ${rep.ungradeable.length} cell(s) could not ` + `be graded, so the cells above are whichever ones survived. Re-run before concluding anything.`);
|
|
129588
|
+
}
|
|
129589
|
+
return L.join(`
|
|
129590
|
+
`);
|
|
129591
|
+
}
|
|
129592
|
+
function toCsv(r) {
|
|
129593
|
+
const head = "label,bank,rows,concurrency,n,errors,p50_ms,p95_ms,p95_ci_low_ms,p95_ci_high_ms,p99_ms,max_ms,mean_ms,stddev_ms,cv,mean_results,zero_result_calls,contention";
|
|
129594
|
+
const rows = r.cells.map((c) => [
|
|
129595
|
+
JSON.stringify(r.config.label),
|
|
129596
|
+
c.bank,
|
|
129597
|
+
c.rows,
|
|
129598
|
+
c.concurrency,
|
|
129599
|
+
c.stats.n,
|
|
129600
|
+
c.stats.errors,
|
|
129601
|
+
round(c.stats.p50, 2),
|
|
129602
|
+
round(c.stats.p95, 2),
|
|
129603
|
+
round(c.stats.p95CiLow, 2),
|
|
129604
|
+
round(c.stats.p95CiHigh, 2),
|
|
129605
|
+
round(c.stats.p99, 2),
|
|
129606
|
+
round(c.stats.max, 2),
|
|
129607
|
+
round(c.stats.mean, 2),
|
|
129608
|
+
round(c.stats.stddev, 2),
|
|
129609
|
+
round(c.stats.cv, 4),
|
|
129610
|
+
round(c.meanResults, 2),
|
|
129611
|
+
c.zeroResultCalls,
|
|
129612
|
+
r.config.contention
|
|
129613
|
+
].join(","));
|
|
129614
|
+
return [head, ...rows].join(`
|
|
129615
|
+
`);
|
|
129616
|
+
}
|
|
129617
|
+
|
|
129618
|
+
// src/hindsight-bench/run.ts
|
|
129619
|
+
var MAX_ERROR_SAMPLES = 5;
|
|
129620
|
+
var realSleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
129621
|
+
async function driveCell(bank, concurrency, count, recall) {
|
|
129622
|
+
const out = [];
|
|
129623
|
+
let cursor = 0;
|
|
129624
|
+
const worker = async () => {
|
|
129625
|
+
for (;; ) {
|
|
129626
|
+
const i2 = cursor++;
|
|
129627
|
+
if (i2 >= count)
|
|
129628
|
+
return;
|
|
129629
|
+
const query = RECALL_QUERIES[i2 % RECALL_QUERIES.length];
|
|
129630
|
+
out.push(await recall(bank, query));
|
|
129631
|
+
}
|
|
129632
|
+
};
|
|
129633
|
+
await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()));
|
|
129634
|
+
return out;
|
|
129635
|
+
}
|
|
129636
|
+
async function runSweep(opts) {
|
|
129637
|
+
const { config: config2, db } = opts;
|
|
129638
|
+
const deps = opts.deps ?? {};
|
|
129639
|
+
const sleep5 = deps.sleep ?? realSleep;
|
|
129640
|
+
const log = deps.log ?? (() => {});
|
|
129641
|
+
const settleMs = opts.settleMs ?? 2000;
|
|
129642
|
+
const recall = deps.recall ?? ((bank, query) => recallOnce(bank, query, {
|
|
129643
|
+
apiUrl: config2.apiUrl,
|
|
129644
|
+
timeoutMs: config2.timeoutMs,
|
|
129645
|
+
budget: config2.budget,
|
|
129646
|
+
maxTokens: config2.maxTokens
|
|
129647
|
+
}));
|
|
129648
|
+
const rowsFor = new Map(db.bankRows.map((b) => [b.bank, b.rows]));
|
|
129649
|
+
const cells = [];
|
|
129650
|
+
for (const bank of config2.banks) {
|
|
129651
|
+
for (const concurrency of [...config2.concurrency].sort((a, b) => a - b)) {
|
|
129652
|
+
if (config2.warmup > 0) {
|
|
129653
|
+
await driveCell(bank, concurrency, config2.warmup, recall);
|
|
129654
|
+
}
|
|
129655
|
+
const raw = await driveCell(bank, concurrency, config2.samples, recall);
|
|
129656
|
+
const ok = raw.filter((s) => s.ok);
|
|
129657
|
+
const okMs = ok.map((s) => s.ms);
|
|
129658
|
+
const errs = raw.filter((s) => !s.ok);
|
|
129659
|
+
const errorSamples = [...new Set(errs.map((e) => e.error ?? "unknown"))].slice(0, MAX_ERROR_SAMPLES);
|
|
129660
|
+
const stats = summarize(okMs, errs.length);
|
|
129661
|
+
cells.push({
|
|
129662
|
+
bank,
|
|
129663
|
+
rows: rowsFor.get(bank) ?? 0,
|
|
129664
|
+
concurrency,
|
|
129665
|
+
stats,
|
|
129666
|
+
meanResults: ok.length === 0 ? 0 : ok.reduce((a, s) => a + s.results, 0) / ok.length,
|
|
129667
|
+
zeroResultCalls: ok.filter((s) => s.results === 0).length,
|
|
129668
|
+
samplesMs: okMs,
|
|
129669
|
+
errorSamples
|
|
129670
|
+
});
|
|
129671
|
+
log(` ${bank} c=${concurrency}: n=${stats.n} err=${stats.errors} ` + `p50=${stats.p50.toFixed(0)}ms p95=${stats.p95.toFixed(0)}ms p99=${stats.p99.toFixed(0)}ms`);
|
|
129672
|
+
if (settleMs > 0)
|
|
129673
|
+
await sleep5(settleMs);
|
|
129674
|
+
}
|
|
129675
|
+
}
|
|
129676
|
+
return cells;
|
|
129677
|
+
}
|
|
129678
|
+
function extractArms(trace) {
|
|
129679
|
+
const rr = trace?.retrieval_results;
|
|
129680
|
+
if (!Array.isArray(rr))
|
|
129681
|
+
return [];
|
|
129682
|
+
const out = [];
|
|
129683
|
+
for (const entry of rr) {
|
|
129684
|
+
const method = typeof entry?.method_name === "string" ? entry.method_name : null;
|
|
129685
|
+
const factType = typeof entry?.fact_type === "string" ? entry.fact_type : "";
|
|
129686
|
+
const secs = Number(entry?.duration_seconds);
|
|
129687
|
+
if (method === null || !Number.isFinite(secs))
|
|
129688
|
+
continue;
|
|
129689
|
+
out.push({ method, fact_type: factType, ms: secs * 1000 });
|
|
129690
|
+
}
|
|
129691
|
+
return out;
|
|
129692
|
+
}
|
|
129693
|
+
async function runArmSweep(opts) {
|
|
129694
|
+
const { config: config2 } = opts;
|
|
129695
|
+
const log = opts.deps?.log ?? (() => {});
|
|
129696
|
+
const recall = opts.deps?.recall ?? ((bank, query) => recallOnce(bank, query, {
|
|
129697
|
+
apiUrl: config2.apiUrl,
|
|
129698
|
+
timeoutMs: config2.timeoutMs,
|
|
129699
|
+
budget: config2.budget,
|
|
129700
|
+
maxTokens: config2.maxTokens,
|
|
129701
|
+
trace: true
|
|
129702
|
+
}));
|
|
129703
|
+
const buckets = new Map;
|
|
129704
|
+
for (const bank of config2.banks) {
|
|
129705
|
+
for (let i2 = 0;i2 < opts.samples; i2++) {
|
|
129706
|
+
const query = RECALL_QUERIES[i2 % RECALL_QUERIES.length];
|
|
129707
|
+
const sample = await recall(bank, query);
|
|
129708
|
+
if (!sample.ok)
|
|
129709
|
+
continue;
|
|
129710
|
+
for (const arm of extractArms(sample.trace)) {
|
|
129711
|
+
const key = `${bank}\x00${arm.method}\x00${arm.fact_type}`;
|
|
129712
|
+
const b = buckets.get(key) ?? { bank, method: arm.method, fact_type: arm.fact_type, ms: [] };
|
|
129713
|
+
b.ms.push(arm.ms);
|
|
129714
|
+
buckets.set(key, b);
|
|
129715
|
+
}
|
|
129716
|
+
}
|
|
129717
|
+
log(` arms: ${bank} traced ${opts.samples}x`);
|
|
129718
|
+
}
|
|
129719
|
+
const rows = [];
|
|
129720
|
+
for (const b of buckets.values()) {
|
|
129721
|
+
const s = summarize(b.ms, 0);
|
|
129722
|
+
rows.push({ bank: b.bank, method: b.method, fact_type: b.fact_type, n: s.n, p50: s.p50, p95: s.p95, max: s.max });
|
|
129723
|
+
}
|
|
129724
|
+
rows.sort((a, b) => a.bank.localeCompare(b.bank) || b.p50 - a.p50);
|
|
129725
|
+
return rows;
|
|
129726
|
+
}
|
|
129727
|
+
|
|
129728
|
+
// src/hindsight-bench/types.ts
|
|
129729
|
+
var BENCH_SCHEMA_VERSION = 2;
|
|
129730
|
+
|
|
129731
|
+
// src/cli/hindsight-bench.ts
|
|
129732
|
+
var DEFAULT_API_URL = "http://127.0.0.1:18888";
|
|
129733
|
+
function fail9(msg) {
|
|
129734
|
+
process.stderr.write(source_default.red(`hindsight-bench: ${msg}
|
|
129735
|
+
`));
|
|
129736
|
+
process.exit(2);
|
|
129737
|
+
}
|
|
129738
|
+
function readResult(path10) {
|
|
129739
|
+
let parsed;
|
|
129740
|
+
try {
|
|
129741
|
+
parsed = JSON.parse(readFileSync113(path10, "utf8"));
|
|
129742
|
+
} catch (e) {
|
|
129743
|
+
fail9(`cannot read ${path10}: ${e.message}`);
|
|
129744
|
+
}
|
|
129745
|
+
if (!Array.isArray(parsed.cells) || parsed.config === undefined) {
|
|
129746
|
+
fail9(`${path10} is not a hindsight-bench result file (no cells/config)`);
|
|
129747
|
+
}
|
|
129748
|
+
if (parsed.schema !== BENCH_SCHEMA_VERSION) {
|
|
129749
|
+
fail9(`${path10} is schema v${parsed.schema}, this build reads v${BENCH_SCHEMA_VERSION}`);
|
|
129750
|
+
}
|
|
129751
|
+
const dupes = duplicateCellKeys(parsed.cells);
|
|
129752
|
+
if (dupes.length > 0) {
|
|
129753
|
+
fail9(`${path10} has duplicate cells (${dupes.join(", ")}) \u2014 cannot be graded`);
|
|
129754
|
+
}
|
|
129755
|
+
return parsed;
|
|
129756
|
+
}
|
|
129757
|
+
function writeOut(path10, body) {
|
|
129758
|
+
const abs = resolve67(path10);
|
|
129759
|
+
mkdirSync71(dirname51(abs), { recursive: true });
|
|
129760
|
+
writeFileSync52(abs, body);
|
|
129761
|
+
process.stderr.write(`${source_default.green("\u2713")} wrote ${abs}
|
|
129762
|
+
`);
|
|
129763
|
+
}
|
|
129764
|
+
function resolveProfile(v) {
|
|
129765
|
+
if (v === undefined || v === false)
|
|
129766
|
+
return "off";
|
|
129767
|
+
if (v === true || v === "")
|
|
129768
|
+
return "read";
|
|
129769
|
+
if (v === "off" || v === "read" || v === "write")
|
|
129770
|
+
return v;
|
|
129771
|
+
fail9(`--contention must be one of off|read|write (got "${String(v)}")`);
|
|
129772
|
+
}
|
|
129773
|
+
function intOpt(raw, name) {
|
|
129774
|
+
const n = Number(raw);
|
|
129775
|
+
if (!Number.isInteger(n) || n < 0)
|
|
129776
|
+
fail9(`${name} must be a non-negative integer (got "${raw}")`);
|
|
129777
|
+
return n;
|
|
129778
|
+
}
|
|
129779
|
+
function registerHindsightBenchCommand(program3) {
|
|
129780
|
+
program3.command("hindsight-bench").description("Measure Hindsight recall latency as a function of bank size and concurrency. " + "Percentiles, not means. Read-only by default; --contention degrades the live box.").option("--api-url <url>", "Hindsight REST base", DEFAULT_API_URL).option("--container <name>", "hindsight container for the psql probes", DEFAULT_CONTAINER2).option("--banks <spec>", "all | top:<n> | spread:<n> | comma list", "spread:5").option("--concurrency <list>", "comma-separated concurrency levels", "1,4,8,16").option("--samples <n>", "recorded recalls per cell", "60").option("--warmup <n>", "discarded recalls per cell before recording", "8").option("--timeout-ms <ms>", "per-recall timeout", "30000").option("--settle-ms <ms>", "quiet period between cells", "2000").option("--budget <budget>", "recall budget", "mid").option("--max-tokens <n>", "recall max_tokens", "4096").option("--contention [profile]", "run the sweep under synthetic load: read (cache churn, SELECT-only) or " + "write (adds a WAL storm against a harness-owned scratch table; needs --allow-writes)").option("--contention-workers <n>", "concurrent load backends", String(DEFAULT_CONTENTION.workers)).option("--contention-scan-pct <pct>", "TABLESAMPLE percentage per churn scan", String(DEFAULT_CONTENTION.scanPct)).option("--contention-max-seconds <s>", "absolute in-SQL deadline for every load backend (orphan guard)", String(DEFAULT_CONTENTION.maxSeconds)).option("--reset-stats", "call pg_stat_reset() before the sweep (never implicit)", false).option("--allow-writes", "authorise a writable database session (AC5 gate)", false).option("--arms [n]", "additionally run a traced per-arm attribution pass (n samples/bank, default 5)").option("--label <text>", "free-form label recorded in the result file", "").option("--out <path>", "write the JSON result file (or the SVG in --plot mode)").option("--csv <path>", "also write a flat per-cell CSV").option("--plot <files...>", "render the chart from result files instead of measuring").option("--compare <files...>", "AC1 reproducibility verdict over two result files").option("--contention-compare <files...>", "AC4 contention verdict over two result files").option("--tolerance <fraction>", "AC1 tolerance", String(DEFAULT_TOLERANCE)).option("--json", "emit machine-readable JSON for the verdict modes", false).action(async (opts) => {
|
|
129781
|
+
if (opts.plot !== undefined)
|
|
129782
|
+
return runPlotMode(opts);
|
|
129783
|
+
if (opts.compare !== undefined)
|
|
129784
|
+
return runCompareMode(opts);
|
|
129785
|
+
if (opts.contentionCompare !== undefined)
|
|
129786
|
+
return runContentionCompareMode(opts);
|
|
129787
|
+
await runMeasureMode(opts);
|
|
129788
|
+
});
|
|
129789
|
+
}
|
|
129790
|
+
function expectTwo(files, flag) {
|
|
129791
|
+
if (files.length !== 2)
|
|
129792
|
+
fail9(`${flag} takes exactly two result files (got ${files.length})`);
|
|
129793
|
+
return [files[0], files[1]];
|
|
129794
|
+
}
|
|
129795
|
+
function runPlotMode(opts) {
|
|
129796
|
+
const results = opts.plot.map(readResult);
|
|
129797
|
+
const svg = renderPlot(results);
|
|
129798
|
+
if (opts.out === undefined) {
|
|
129799
|
+
process.stdout.write(svg + `
|
|
129800
|
+
`);
|
|
129801
|
+
return;
|
|
129802
|
+
}
|
|
129803
|
+
writeOut(opts.out, svg + `
|
|
129804
|
+
`);
|
|
129805
|
+
}
|
|
129806
|
+
function runCompareMode(opts) {
|
|
129807
|
+
const [pa, pb] = expectTwo(opts.compare, "--compare");
|
|
129808
|
+
const tolerance = Number(opts.tolerance);
|
|
129809
|
+
if (!Number.isFinite(tolerance) || tolerance < 0)
|
|
129810
|
+
fail9(`--tolerance must be a non-negative number`);
|
|
129811
|
+
const rep = compareRuns(readResult(pa), readResult(pb), tolerance);
|
|
129812
|
+
process.stdout.write((opts.json ? JSON.stringify(rep, null, 2) : formatReproducibility(rep)) + `
|
|
129813
|
+
`);
|
|
129814
|
+
if (!rep.pass)
|
|
129815
|
+
process.exitCode = 1;
|
|
129816
|
+
}
|
|
129817
|
+
function runContentionCompareMode(opts) {
|
|
129818
|
+
const [pi, pl] = expectTwo(opts.contentionCompare, "--contention-compare");
|
|
129819
|
+
const rep = compareContention(readResult(pi), readResult(pl));
|
|
129820
|
+
process.stdout.write((opts.json ? JSON.stringify(rep, null, 2) : formatContention(rep)) + `
|
|
129821
|
+
`);
|
|
129822
|
+
if (!rep.pass)
|
|
129823
|
+
process.exitCode = 1;
|
|
129824
|
+
}
|
|
129825
|
+
async function runMeasureMode(opts) {
|
|
129826
|
+
const profile = resolveProfile(opts.contention);
|
|
129827
|
+
const container = opts.container;
|
|
129828
|
+
const sqlOpts = { container };
|
|
129829
|
+
if (profile === "write" && !opts.allowWrites) {
|
|
129830
|
+
fail9("--contention write opens a writable session for its scratch table; pass --allow-writes to authorise it");
|
|
129831
|
+
}
|
|
129832
|
+
try {
|
|
129833
|
+
assertReadOnlyOrWritesAllowed(opts.allowWrites, sqlOpts);
|
|
129834
|
+
} catch (e) {
|
|
129835
|
+
fail9(e.message);
|
|
129836
|
+
}
|
|
129837
|
+
let db;
|
|
129838
|
+
try {
|
|
129839
|
+
db = readDbState(sqlOpts);
|
|
129840
|
+
} catch (e) {
|
|
129841
|
+
fail9(`could not read database state: ${e.message}`);
|
|
129842
|
+
}
|
|
129843
|
+
const instance = readInstanceState(sqlOpts);
|
|
129844
|
+
let banks;
|
|
129845
|
+
let concurrency;
|
|
129846
|
+
try {
|
|
129847
|
+
banks = selectBanks(db.bankRows, opts.banks);
|
|
129848
|
+
concurrency = parseConcurrency(opts.concurrency);
|
|
129849
|
+
} catch (e) {
|
|
129850
|
+
if (e instanceof BankSelectionError)
|
|
129851
|
+
fail9(e.message);
|
|
129852
|
+
throw e;
|
|
129853
|
+
}
|
|
129854
|
+
if (opts.resetStats) {
|
|
129855
|
+
try {
|
|
129856
|
+
resetStats(sqlOpts);
|
|
129857
|
+
process.stderr.write(`${source_default.yellow("!")} pg_stat_reset() called \u2014 cumulative statistics were discarded
|
|
129858
|
+
`);
|
|
129859
|
+
} catch (e) {
|
|
129860
|
+
fail9(`--reset-stats failed: ${e.message}`);
|
|
129861
|
+
}
|
|
129862
|
+
}
|
|
129863
|
+
const config2 = {
|
|
129864
|
+
startedAt: new Date().toISOString(),
|
|
129865
|
+
apiUrl: opts.apiUrl,
|
|
129866
|
+
container,
|
|
129867
|
+
banks,
|
|
129868
|
+
concurrency,
|
|
129869
|
+
samples: intOpt(opts.samples, "--samples"),
|
|
129870
|
+
warmup: intOpt(opts.warmup, "--warmup"),
|
|
129871
|
+
timeoutMs: intOpt(opts.timeoutMs, "--timeout-ms"),
|
|
129872
|
+
contention: profile,
|
|
129873
|
+
contentionWorkers: profile === "off" ? 0 : intOpt(opts.contentionWorkers, "--contention-workers"),
|
|
129874
|
+
statsReset: opts.resetStats,
|
|
129875
|
+
allowWrites: opts.allowWrites,
|
|
129876
|
+
querySet: QUERY_SET_ID,
|
|
129877
|
+
budget: opts.budget,
|
|
129878
|
+
maxTokens: intOpt(opts.maxTokens, "--max-tokens"),
|
|
129879
|
+
label: opts.label
|
|
129880
|
+
};
|
|
129881
|
+
const armSamples = opts.arms === undefined ? 0 : opts.arms === true || opts.arms === "" ? 5 : Number(opts.arms);
|
|
129882
|
+
if (!Number.isFinite(armSamples) || armSamples < 0)
|
|
129883
|
+
fail9(`--arms must be a non-negative integer`);
|
|
129884
|
+
process.stderr.write(`sweeping ${banks.length} bank(s) \u00d7 ${concurrency.length} concurrency level(s) = ` + `${banks.length * concurrency.length} cells, ${config2.samples} samples each ` + `(+${config2.warmup} warm-up) \u00b7 contention=${profile}
|
|
129885
|
+
`);
|
|
129886
|
+
const t0 = Date.now();
|
|
129887
|
+
let load = noContention();
|
|
129888
|
+
let result;
|
|
129889
|
+
try {
|
|
129890
|
+
if (profile !== "off") {
|
|
129891
|
+
load = await startContention({
|
|
129892
|
+
profile,
|
|
129893
|
+
workers: config2.contentionWorkers,
|
|
129894
|
+
scanPct: Number(opts.contentionScanPct),
|
|
129895
|
+
maxSeconds: intOpt(opts.contentionMaxSeconds, "--contention-max-seconds"),
|
|
129896
|
+
container
|
|
129897
|
+
});
|
|
129898
|
+
process.stderr.write(`${source_default.yellow("!")} contention "${profile}" running: ${load.liveBackends} of ` + `${load.workers} worker(s) confirmed attached to PostgreSQL \u2014 ` + `the live fleet is degraded until this finishes
|
|
129899
|
+
`);
|
|
129900
|
+
const onSignal = () => {
|
|
129901
|
+
load.stop();
|
|
129902
|
+
process.exit(130);
|
|
129903
|
+
};
|
|
129904
|
+
process.once("SIGINT", onSignal);
|
|
129905
|
+
process.once("SIGTERM", onSignal);
|
|
129906
|
+
await new Promise((r) => setTimeout(r, 5000));
|
|
129907
|
+
}
|
|
129908
|
+
const cells = await runSweep({
|
|
129909
|
+
config: config2,
|
|
129910
|
+
db,
|
|
129911
|
+
settleMs: intOpt(opts.settleMs, "--settle-ms"),
|
|
129912
|
+
deps: { log: (m) => process.stderr.write(`${m}
|
|
129913
|
+
`) }
|
|
129914
|
+
});
|
|
129915
|
+
const arms = armSamples > 0 ? await runArmSweep({ config: config2, samples: armSamples, deps: { log: (m) => process.stderr.write(`${m}
|
|
129916
|
+
`) } }) : null;
|
|
129917
|
+
result = {
|
|
129918
|
+
schema: BENCH_SCHEMA_VERSION,
|
|
129919
|
+
config: config2,
|
|
129920
|
+
db,
|
|
129921
|
+
instance,
|
|
129922
|
+
cells,
|
|
129923
|
+
arms,
|
|
129924
|
+
durationS: (Date.now() - t0) / 1000
|
|
129925
|
+
};
|
|
129926
|
+
} finally {
|
|
129927
|
+
load.stop();
|
|
129928
|
+
}
|
|
129929
|
+
process.stdout.write(formatSummary(result) + `
|
|
129930
|
+
`);
|
|
129931
|
+
const { result: safeResult, mapping } = anonymiseResult(result);
|
|
129932
|
+
if (opts.out !== undefined || opts.csv !== undefined) {
|
|
129933
|
+
process.stderr.write(`${source_default.cyan("i")} bank names are pseudonymised in the written file(s); this mapping is NOT persisted:
|
|
129934
|
+
` + `${formatBankMapping(mapping)}
|
|
129935
|
+
`);
|
|
129936
|
+
}
|
|
129937
|
+
if (opts.out !== undefined)
|
|
129938
|
+
writeOut(opts.out, JSON.stringify(safeResult, null, 2) + `
|
|
129939
|
+
`);
|
|
129940
|
+
if (opts.csv !== undefined)
|
|
129941
|
+
writeOut(opts.csv, toCsv(safeResult) + `
|
|
129942
|
+
`);
|
|
129943
|
+
if (opts.out === undefined && opts.csv === undefined) {
|
|
129944
|
+
process.stderr.write(source_default.yellow(`! no --out given \u2014 this run's samples were not persisted and cannot be diffed later
|
|
129945
|
+
`));
|
|
129946
|
+
}
|
|
129947
|
+
}
|
|
129948
|
+
|
|
128567
129949
|
// src/cli/openrouter-watch.ts
|
|
128568
129950
|
init_source();
|
|
128569
129951
|
import { existsSync as existsSync127, readdirSync as readdirSync51 } from "node:fs";
|
|
128570
129952
|
import { homedir as homedir71 } from "node:os";
|
|
128571
|
-
import { join as join126, resolve as
|
|
129953
|
+
import { join as join126, resolve as resolve69 } from "node:path";
|
|
128572
129954
|
|
|
128573
129955
|
// src/openrouter/install-cron.ts
|
|
128574
|
-
import { existsSync as existsSync126, mkdirSync as
|
|
128575
|
-
import { dirname as
|
|
129956
|
+
import { existsSync as existsSync126, mkdirSync as mkdirSync72, readFileSync as readFileSync114, renameSync as renameSync33, writeFileSync as writeFileSync53 } from "node:fs";
|
|
129957
|
+
import { dirname as dirname52 } from "node:path";
|
|
128576
129958
|
var CRON_PATH3 = "/etc/cron.d/openrouter-watch";
|
|
128577
129959
|
var CRON_SCHEDULE2 = "17 * * * *";
|
|
128578
129960
|
var CRON_LOG_PATH3 = "/var/log/openrouter-watch.log";
|
|
@@ -128591,28 +129973,28 @@ function installCron3(opts) {
|
|
|
128591
129973
|
const content = renderCron3(opts);
|
|
128592
129974
|
if (existsSync126(path10)) {
|
|
128593
129975
|
try {
|
|
128594
|
-
if (
|
|
129976
|
+
if (readFileSync114(path10, "utf8") === content) {
|
|
128595
129977
|
return { status: "unchanged", path: path10, content };
|
|
128596
129978
|
}
|
|
128597
129979
|
} catch {}
|
|
128598
129980
|
}
|
|
128599
|
-
|
|
129981
|
+
mkdirSync72(dirname52(path10), { recursive: true });
|
|
128600
129982
|
const tmp = `${path10}.${process.pid}.tmp`;
|
|
128601
|
-
|
|
129983
|
+
writeFileSync53(tmp, content, { mode: 420 });
|
|
128602
129984
|
renameSync33(tmp, path10);
|
|
128603
129985
|
return { status: "installed", path: path10, content };
|
|
128604
129986
|
}
|
|
128605
129987
|
// src/openrouter/state.ts
|
|
128606
|
-
import { mkdirSync as
|
|
129988
|
+
import { mkdirSync as mkdirSync73, readFileSync as readFileSync115, renameSync as renameSync34, writeFileSync as writeFileSync54 } from "node:fs";
|
|
128607
129989
|
import { homedir as homedir70 } from "node:os";
|
|
128608
|
-
import { dirname as
|
|
129990
|
+
import { dirname as dirname53, resolve as resolve68 } from "node:path";
|
|
128609
129991
|
function defaultStatePath3(home2 = process.env.SWITCHROOM_HOME ?? process.env.HOME ?? homedir70()) {
|
|
128610
|
-
return
|
|
129992
|
+
return resolve68(home2, ".switchroom", "openrouter-watch", "state.json");
|
|
128611
129993
|
}
|
|
128612
129994
|
function loadState2(path10) {
|
|
128613
129995
|
let parsed;
|
|
128614
129996
|
try {
|
|
128615
|
-
parsed = JSON.parse(
|
|
129997
|
+
parsed = JSON.parse(readFileSync115(path10, "utf8"));
|
|
128616
129998
|
} catch {
|
|
128617
129999
|
return {};
|
|
128618
130000
|
}
|
|
@@ -128632,10 +130014,10 @@ function loadState2(path10) {
|
|
|
128632
130014
|
return out;
|
|
128633
130015
|
}
|
|
128634
130016
|
function saveState2(path10, state) {
|
|
128635
|
-
|
|
130017
|
+
mkdirSync73(dirname53(path10), { recursive: true, mode: 448 });
|
|
128636
130018
|
const tmp = `${path10}.${process.pid}.tmp`;
|
|
128637
130019
|
const payload = { v: 1, ...state };
|
|
128638
|
-
|
|
130020
|
+
writeFileSync54(tmp, JSON.stringify(payload, null, 2) + `
|
|
128639
130021
|
`, { mode: 384 });
|
|
128640
130022
|
renameSync34(tmp, path10);
|
|
128641
130023
|
}
|
|
@@ -128821,7 +130203,7 @@ async function notifyOperator2(agentsDir, text, log) {
|
|
|
128821
130203
|
const candidates = [];
|
|
128822
130204
|
try {
|
|
128823
130205
|
for (const name of readdirSync51(agentsDir).sort()) {
|
|
128824
|
-
const sock =
|
|
130206
|
+
const sock = resolve69(agentsDir, name, "telegram", "gateway.sock");
|
|
128825
130207
|
if (existsSync127(sock))
|
|
128826
130208
|
candidates.push({ agent: name, sock });
|
|
128827
130209
|
}
|
|
@@ -129079,6 +130461,7 @@ registerWebdCommand(program3);
|
|
|
129079
130461
|
registerHostCommand(program3);
|
|
129080
130462
|
registerFleetHealthCommand(program3);
|
|
129081
130463
|
registerHindsightWatchCommand(program3);
|
|
130464
|
+
registerHindsightBenchCommand(program3);
|
|
129082
130465
|
registerOpenRouterWatchCommand(program3);
|
|
129083
130466
|
registerConfigRepoCommand(program3);
|
|
129084
130467
|
|