switchroom 0.19.27 → 0.19.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +5 -2
- package/dist/auth-broker/index.js +129 -8
- package/dist/cli/autoaccept-poll.js +225 -17
- package/dist/cli/notion-write-pretool.mjs +5 -2
- package/dist/cli/switchroom.js +796 -35
- package/dist/host-control/main.js +130 -9
- package/dist/vault/approvals/kernel-server.js +129 -8
- package/dist/vault/broker/server.js +129 -8
- package/package.json +3 -2
- package/profiles/_base/start.sh.hbs +70 -15
- package/telegram-plugin/dist/bridge/bridge.js +1 -0
- package/telegram-plugin/dist/gateway/gateway.js +568 -49
- package/telegram-plugin/dist/server.js +1 -0
- package/telegram-plugin/edit-flood-fuse.ts +230 -27
- package/telegram-plugin/gateway/callback-query-handlers.ts +6 -0
- package/telegram-plugin/gateway/gateway.ts +9 -2
- package/telegram-plugin/gateway/mcp-failure-hook.ts +74 -0
- package/telegram-plugin/inline-keyboard-callbacks.ts +202 -21
- package/telegram-plugin/mcp-credential-failure.ts +459 -0
- package/telegram-plugin/operator-events.ts +38 -0
- package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +58 -1
- package/telegram-plugin/tests/edit-flood-fuse-reply-reserve.test.ts +340 -0
- package/telegram-plugin/tests/finalize-callback-flood-policy.test.ts +298 -0
- package/telegram-plugin/tests/finalize-callback.test.ts +41 -8
- package/telegram-plugin/tests/mcp-credential-failure.test.ts +310 -0
- package/vendor/hindsight-memory/scripts/drain_pending.py +433 -11
- package/vendor/hindsight-memory/scripts/lib/pending.py +193 -28
- package/vendor/hindsight-memory/scripts/tests/test_drain_circuit_breaker.py +401 -0
- package/vendor/hindsight-memory/scripts/tests/test_drain_serialisation.py +286 -0
- package/vendor/hindsight-memory/scripts/tests/test_pending_drops.py +817 -8
- package/vendor/hindsight-memory/settings.json +1 -1
- package/vendor/hindsight-memory/tests/test_hooks.py +11 -2
package/dist/cli/switchroom.js
CHANGED
|
@@ -2120,7 +2120,7 @@ var init_esm = __esm(() => {
|
|
|
2120
2120
|
});
|
|
2121
2121
|
|
|
2122
2122
|
// src/build-info.ts
|
|
2123
|
-
var VERSION = "0.19.
|
|
2123
|
+
var VERSION = "0.19.28", COMMIT_SHA = "d9a5f4ae";
|
|
2124
2124
|
|
|
2125
2125
|
// src/cli/resolve-version.ts
|
|
2126
2126
|
import { existsSync, readFileSync } from "node:fs";
|
|
@@ -13966,7 +13966,10 @@ var init_schema = __esm(() => {
|
|
|
13966
13966
|
admin_key: exports_external.string().optional().describe("LiteLLM master/admin key used at apply time to provision the team + " + "virtual key. Supports a vault reference (e.g. " + "'vault:litellm/master-key') \u2014 resolution happens at apply time via " + "the vault-broker. Never injected into the agent container."),
|
|
13967
13967
|
team: exports_external.string().optional().describe("LiteLLM team alias the per-agent key is created under. Defaults to " + "'switchroom' (applied in code, not as a schema default)."),
|
|
13968
13968
|
small_fast_model: exports_external.string().optional().describe("Model id exported as ANTHROPIC_SMALL_FAST_MODEL for the claude CLI's " + "background/fast lane, e.g. 'claude-haiku-4-5-20251001'."),
|
|
13969
|
-
tags: exports_external.record(exports_external.string(), exports_external.string()).optional().describe("Extra key/value metadata tags attached to the provisioned LiteLLM " + "virtual key. Merged per-key across cascade layers (agent wins).")
|
|
13969
|
+
tags: exports_external.record(exports_external.string(), exports_external.string()).optional().describe("Extra key/value metadata tags attached to the provisioned LiteLLM " + "virtual key. Merged per-key across cascade layers (agent wins)."),
|
|
13970
|
+
max_budget: exports_external.number().positive().optional().describe("HARD spend cap in USD for this agent's virtual key over one " + "`budget_duration` window. LiteLLM refuses the request once the key's " + "tracked spend exceeds it, so a runaway loop costs at most this much " + "before it is stopped. Defaults to " + "DEFAULT_KEY_MAX_BUDGET_USD (see src/litellm/budget.ts) \u2014 deliberately " + "conservative; raise it per-agent rather than removing it. Set 0 or " + "omit `budget_duration` at your own risk: an uncapped key is only as " + "bounded as the upstream account balance."),
|
|
13971
|
+
soft_budget: exports_external.number().positive().optional().describe("ADVISORY spend threshold in USD. LiteLLM keeps serving past it and " + "raises a budget alert instead. Must be < max_budget. NOTE: LiteLLM " + "accepts soft_budget only on POST /key/generate (GenerateKeyRequest); " + "UpdateKeyRequest does NOT carry it, so changing this value only takes " + "effect on a key that is (re)generated, not on an existing one."),
|
|
13972
|
+
budget_duration: exports_external.string().regex(/^\d+(s|m|h|d|mo)$/, "budget_duration must be a LiteLLM duration like '30d', '24h', '1mo'").optional().describe("Rolling window the budget resets on, in LiteLLM duration syntax " + "('30d', '24h', '1mo'). Defaults to DEFAULT_KEY_BUDGET_DURATION. " + "WITHOUT a duration LiteLLM treats max_budget as a LIFETIME cap that " + "never resets \u2014 the key silently dies for good once it is hit.")
|
|
13970
13973
|
}).optional().describe("LiteLLM routing config \u2014 opt-in per-agent virtual-key auto-provisioning " + "+ routing env. Default OFF. See LiteLLMConfigSchema doc for the full flow.");
|
|
13971
13974
|
HindsightPerOpLlmSchema = exports_external.object({
|
|
13972
13975
|
model: exports_external.string().min(1).optional().describe("Per-op model (upstream `HINDSIGHT_API_<OP>_LLM_MODEL`). Absent \u2192 " + "inherit the global `hindsight.llm.model`."),
|
|
@@ -13985,7 +13988,7 @@ var init_schema = __esm(() => {
|
|
|
13985
13988
|
reflect: HindsightPerOpLlmSchema.optional().describe("Per-op override for the `reflect` LLM op (synthesis / mental-model " + "refresh). Emits `HINDSIGHT_API_REFLECT_LLM_*`. Absent \u2192 uses global."),
|
|
13986
13989
|
consolidation: HindsightPerOpLlmSchema.optional().describe("Per-op override for the `consolidation` LLM op (background memory " + "merge). Emits `HINDSIGHT_API_CONSOLIDATION_LLM_*`. Absent \u2192 global.")
|
|
13987
13990
|
}).optional().describe("LLM knob for the hindsight container. The flat `provider`/`model` set " + "the global default (backward-compatible); optional `retain`/`reflect`/" + "`consolidation` blocks override individual ops. All fields optional; " + "unset fields fall back to the hard-coded defaults."),
|
|
13988
|
-
env: exports_external.record(exports_external.union([exports_external.string(), exports_external.number(), exports_external.boolean()])).optional().describe("Operator overrides for switchroom's capability-gated Hindsight " + "performance defaults. Only the keys switchroom actually manages are " + "honoured (`HINDSIGHT_PERF_ENV_KEYS` in " + "src/setup/hindsight-perf-defaults.ts: RERANKER_LOCAL_FP16, " + "RERANKER_LOCAL_BATCH_SIZE, LLM_MAX_CONCURRENT, " + "RETAIN/CONSOLIDATION_LLM_MAX_CONCURRENT, LLM_STRICT_SCHEMA, " + "LLM_MAX_RETRIES, CONSOLIDATION_LLM_PARALLELISM, " + "MAX_OBSERVATIONS_PER_SCOPE, " + "RECALL_MAX_CANDIDATES_PER_SOURCE, LINK_EXPANSION_PER_ENTITY_LIMIT, " + "LINK_EXPANSION_TIMEOUT, LLM_REASONING_EFFORT, " + "RERANKER_LOCAL_BUCKET_BATCHING, RERANKER_MAX_CANDIDATES, " + "RERANKER_LOCAL_MAX_CONCURRENT, RECALL_MAX_CONCURRENT, " + "REFLECT_WALL_TIMEOUT, WORKER_CONSOLIDATION_MAX_SLOTS, " + "WORKER_CONSOLIDATION_SLOT_LIMIT, " + "CONSOLIDATION_MAX_MEMORIES_PER_ROUND), the override-only keys " + "switchroom manages but ships NO default for " + "(`HINDSIGHT_PERF_OVERRIDE_ONLY_KEYS`: " + "HINDSIGHT_API_WORKER_CONSOLIDATION_BANK_PRIORITY \u2014 a per-deployment " + "`bank-pattern:priority,...` map; unset means upstream's flat " + "created_at FIFO across banks; and " + "HINDSIGHT_CE_DECISIVE_RELATIVE_GAP \u2014 the rollback knob for " + "switchroom's CE-saturation damping patch, a float; >= ~0.65 backs the " + "damping out entirely, unset means the patch's own derived gap), plus the " + "embedded-PostgreSQL (pg0) sizing keys switchroom manages in " + "src/setup/hindsight-pg-defaults.ts (`HINDSIGHT_PG_ENV_KEYS`: " + "SWITCHROOM_HINDSIGHT_PG_EFFECTIVE_CACHE_SIZE, " + "SWITCHROOM_HINDSIGHT_PG_SHARED_BUFFERS \u2014 a postgres size string such " + "as `4GB`, or the sentinel `off` to leave pg0's own default for that " + "one knob). A value set here " + "REPLACES switchroom's default and is emitted even when the gating " + "capability is absent, so an operator can always force a knob. Other " + "`HINDSIGHT_API_*` keys are deliberately IGNORED \u2014 a blanket " + "passthrough would collide with the vars startHindsight() derives " + "itself (HINDSIGHT_API_PORT, the retain token/deadline budget).")
|
|
13991
|
+
env: exports_external.record(exports_external.union([exports_external.string(), exports_external.number(), exports_external.boolean()])).optional().describe("Operator overrides for switchroom's capability-gated Hindsight " + "performance defaults. Only the keys switchroom actually manages are " + "honoured (`HINDSIGHT_PERF_ENV_KEYS` in " + "src/setup/hindsight-perf-defaults.ts: RERANKER_LOCAL_FP16, " + "RERANKER_LOCAL_BATCH_SIZE, LLM_MAX_CONCURRENT, " + "RETAIN/CONSOLIDATION_LLM_MAX_CONCURRENT, LLM_STRICT_SCHEMA, " + "LLM_MAX_RETRIES, CONSOLIDATION_LLM_PARALLELISM, " + "MAX_OBSERVATIONS_PER_SCOPE, " + "RECALL_MAX_CANDIDATES_PER_SOURCE, LINK_EXPANSION_PER_ENTITY_LIMIT, " + "LINK_EXPANSION_TIMEOUT, LLM_REASONING_EFFORT, " + "RERANKER_LOCAL_BUCKET_BATCHING, RERANKER_MAX_CANDIDATES, " + "RERANKER_LOCAL_MAX_CONCURRENT, RECALL_MAX_CONCURRENT, " + "REFLECT_WALL_TIMEOUT, WORKER_CONSOLIDATION_MAX_SLOTS, " + "WORKER_CONSOLIDATION_SLOT_LIMIT, " + "CONSOLIDATION_MAX_MEMORIES_PER_ROUND, RECENCY_DECAY_FUNCTION, " + "RECENCY_DECAY_HALFLIFE_DAYS \u2014 switchroom defaults recall's recency " + "curve to `exponential` with a 30-day half-life so a fact retained " + "today outranks a stale one, instead of upstream's near-flat " + "linear/365-day window), the override-only keys " + "switchroom manages but ships NO default for " + "(`HINDSIGHT_PERF_OVERRIDE_ONLY_KEYS`: " + "HINDSIGHT_API_WORKER_CONSOLIDATION_BANK_PRIORITY \u2014 a per-deployment " + "`bank-pattern:priority,...` map; unset means upstream's flat " + "created_at FIFO across banks; and " + "HINDSIGHT_CE_DECISIVE_RELATIVE_GAP \u2014 the rollback knob for " + "switchroom's CE-saturation damping patch, a float; >= ~0.65 backs the " + "damping out entirely, unset means the patch's own derived gap; and " + "HINDSIGHT_API_RECENCY_DECAY_LINEAR_WINDOW_DAYS \u2014 only read when the " + "decay function is `linear`, so switchroom ships no default for it but " + "still honours an operator who flips the function back; and " + "HINDSIGHT_API_WORKER_MAX_SLOTS \u2014 the worker poller's TOTAL in-flight " + "task budget, the pool WORKER_CONSOLIDATION_MAX_SLOTS reserves out of; " + "unset means upstream's own default), plus the " + "embedded-PostgreSQL (pg0) sizing keys switchroom manages in " + "src/setup/hindsight-pg-defaults.ts (`HINDSIGHT_PG_ENV_KEYS`: " + "SWITCHROOM_HINDSIGHT_PG_EFFECTIVE_CACHE_SIZE, " + "SWITCHROOM_HINDSIGHT_PG_SHARED_BUFFERS \u2014 a postgres size string such " + "as `4GB`, or the sentinel `off` to leave pg0's own default for that " + "one knob). A value set here " + "REPLACES switchroom's default and is emitted even when the gating " + "capability is absent, so an operator can always force a knob. Other " + "`HINDSIGHT_API_*` keys are deliberately IGNORED \u2014 a blanket " + "passthrough would collide with the vars startHindsight() derives " + "itself (HINDSIGHT_API_PORT, the retain token/deadline budget).")
|
|
13989
13992
|
});
|
|
13990
13993
|
MicrosoftWorkspaceConfigSchema = exports_external.object({
|
|
13991
13994
|
microsoft_client_id: exports_external.string().min(1).optional().describe("Microsoft OAuth application (client) ID from Entra portal " + "(literal string or vault reference e.g. " + "'vault:microsoft-oauth-client-id'). OPTIONAL \u2014 omit it to use " + "switchroom's shipped default Microsoft app (zero-config). " + "Set it only to bring your own Entra app (BYO)."),
|
|
@@ -15765,6 +15768,12 @@ var init_hindsight_pg_defaults = __esm(() => {
|
|
|
15765
15768
|
});
|
|
15766
15769
|
|
|
15767
15770
|
// src/setup/hindsight-perf-defaults.ts
|
|
15771
|
+
function hindsightConsolidationLlmMaxConcurrentDefault(globalMaxConcurrent = HINDSIGHT_DEFAULT_LLM_MAX_CONCURRENT, retainMaxConcurrent = HINDSIGHT_DEFAULT_RETAIN_LLM_MAX_CONCURRENT) {
|
|
15772
|
+
const globalCap = Number.isFinite(globalMaxConcurrent) && globalMaxConcurrent >= 1 ? Math.floor(globalMaxConcurrent) : HINDSIGHT_DEFAULT_LLM_MAX_CONCURRENT;
|
|
15773
|
+
const retainCap = Number.isFinite(retainMaxConcurrent) && retainMaxConcurrent >= 0 ? Math.floor(retainMaxConcurrent) : HINDSIGHT_DEFAULT_RETAIN_LLM_MAX_CONCURRENT;
|
|
15774
|
+
const headroomBound = Math.max(1, globalCap - 1);
|
|
15775
|
+
return Math.min(headroomBound, Math.max(1, globalCap - retainCap - 1));
|
|
15776
|
+
}
|
|
15768
15777
|
function resolveHindsightPerfOverrides(configEnv, processEnv = process.env) {
|
|
15769
15778
|
const out = new Map;
|
|
15770
15779
|
for (const key of HINDSIGHT_PERF_ENV_KEYS) {
|
|
@@ -15791,6 +15800,14 @@ function hindsightPerfEnv(caps, overrides = new Map) {
|
|
|
15791
15800
|
groups.push(HINDSIGHT_PERF_DEFAULTS_GPU);
|
|
15792
15801
|
if (caps.localLlm === true)
|
|
15793
15802
|
groups.push(HINDSIGHT_PERF_DEFAULTS_LOCAL_LLM);
|
|
15803
|
+
const effectiveCap = (key, fallback) => {
|
|
15804
|
+
const raw = overrides.get(key);
|
|
15805
|
+
if (raw === undefined)
|
|
15806
|
+
return fallback;
|
|
15807
|
+
const parsed = Number.parseInt(raw, 10);
|
|
15808
|
+
return Number.isFinite(parsed) && parsed >= 1 ? parsed : fallback;
|
|
15809
|
+
};
|
|
15810
|
+
const derivedConsolidationCap = String(hindsightConsolidationLlmMaxConcurrentDefault(effectiveCap("HINDSIGHT_API_LLM_MAX_CONCURRENT", HINDSIGHT_DEFAULT_LLM_MAX_CONCURRENT), effectiveCap("HINDSIGHT_API_RETAIN_LLM_MAX_CONCURRENT", HINDSIGHT_DEFAULT_RETAIN_LLM_MAX_CONCURRENT)));
|
|
15794
15811
|
const out = [];
|
|
15795
15812
|
const emitted = new Set;
|
|
15796
15813
|
for (const group of groups) {
|
|
@@ -15798,7 +15815,8 @@ function hindsightPerfEnv(caps, overrides = new Map) {
|
|
|
15798
15815
|
if (emitted.has(key))
|
|
15799
15816
|
continue;
|
|
15800
15817
|
emitted.add(key);
|
|
15801
|
-
|
|
15818
|
+
const shipped = key === "HINDSIGHT_API_CONSOLIDATION_LLM_MAX_CONCURRENT" ? derivedConsolidationCap : value;
|
|
15819
|
+
out.push([key, overrides.get(key) ?? shipped]);
|
|
15802
15820
|
}
|
|
15803
15821
|
}
|
|
15804
15822
|
for (const key of [...overrides.keys()].sort()) {
|
|
@@ -15812,10 +15830,11 @@ function hindsightPerfEnv(caps, overrides = new Map) {
|
|
|
15812
15830
|
function findUnmanagedHindsightEnvKeys(configEnv) {
|
|
15813
15831
|
return Object.keys(configEnv ?? {}).filter((key) => !HINDSIGHT_PERF_ENV_KEYS.has(key) && !HINDSIGHT_PG_ENV_KEYS.has(key)).sort();
|
|
15814
15832
|
}
|
|
15815
|
-
var HINDSIGHT_RERANKER_MAX_CANDIDATES_FOR_DERIVATION = 150, HINDSIGHT_DEFAULT_RECALL_MAX_CANDIDATES_PER_SOURCE, HINDSIGHT_DEFAULT_LINK_EXPANSION_PER_ENTITY_LIMIT = 50, HINDSIGHT_DEFAULT_LINK_EXPANSION_TIMEOUT_S = 2, HINDSIGHT_DEFAULT_LLM_REASONING_EFFORT = "low", HINDSIGHT_DEFAULT_LLM_MAX_CONCURRENT = 4, HINDSIGHT_DEFAULT_RETAIN_LLM_MAX_CONCURRENT = 1, HINDSIGHT_DEFAULT_CONSOLIDATION_LLM_MAX_CONCURRENT
|
|
15833
|
+
var HINDSIGHT_RERANKER_MAX_CANDIDATES_FOR_DERIVATION = 150, HINDSIGHT_DEFAULT_RECALL_MAX_CANDIDATES_PER_SOURCE, HINDSIGHT_DEFAULT_LINK_EXPANSION_PER_ENTITY_LIMIT = 50, HINDSIGHT_DEFAULT_LINK_EXPANSION_TIMEOUT_S = 2, HINDSIGHT_DEFAULT_LLM_REASONING_EFFORT = "low", HINDSIGHT_DEFAULT_LLM_MAX_CONCURRENT = 4, HINDSIGHT_DEFAULT_RETAIN_LLM_MAX_CONCURRENT = 1, HINDSIGHT_DEFAULT_CONSOLIDATION_LLM_MAX_CONCURRENT, HINDSIGHT_DEFAULT_RERANKER_LOCAL_FP16 = "true", HINDSIGHT_DEFAULT_RERANKER_LOCAL_BATCH_SIZE = 128, HINDSIGHT_DEFAULT_LLM_STRICT_SCHEMA = "true", HINDSIGHT_DEFAULT_LLM_MAX_RETRIES = 2, HINDSIGHT_DEFAULT_CONSOLIDATION_LLM_PARALLELISM = 2, HINDSIGHT_DEFAULT_MAX_OBSERVATIONS_PER_SCOPE = 1000, HINDSIGHT_DEFAULT_RERANKER_BUCKET_BATCHING = "true", HINDSIGHT_DEFAULT_RERANKER_MAX_CANDIDATES = 150, HINDSIGHT_DEFAULT_RERANKER_LOCAL_MAX_CONCURRENT = 4, HINDSIGHT_DEFAULT_RECALL_MAX_CONCURRENT = 8, HINDSIGHT_DEFAULT_REFLECT_WALL_TIMEOUT_S = 600, HINDSIGHT_DEFAULT_CONSOLIDATION_MAX_MEMORIES_PER_ROUND = 500, HINDSIGHT_DEFAULT_CONSOLIDATION_SLOT_LIMIT = 6, HINDSIGHT_DEFAULT_CONSOLIDATION_MAX_SLOTS = 1, HINDSIGHT_DEFAULT_RECENCY_DECAY_FUNCTION = "exponential", HINDSIGHT_DEFAULT_RECENCY_DECAY_HALFLIFE_DAYS = 30, HINDSIGHT_PERF_DEFAULTS_UNGATED, HINDSIGHT_PERF_DEFAULTS_GPU, HINDSIGHT_PERF_DEFAULTS_LOCAL_LLM, HINDSIGHT_PERF_OVERRIDE_ONLY_KEYS, HINDSIGHT_PERF_ENV_KEYS;
|
|
15816
15834
|
var init_hindsight_perf_defaults = __esm(() => {
|
|
15817
15835
|
init_hindsight_pg_defaults();
|
|
15818
15836
|
HINDSIGHT_DEFAULT_RECALL_MAX_CANDIDATES_PER_SOURCE = Math.ceil(HINDSIGHT_RERANKER_MAX_CANDIDATES_FOR_DERIVATION * 0.4);
|
|
15837
|
+
HINDSIGHT_DEFAULT_CONSOLIDATION_LLM_MAX_CONCURRENT = hindsightConsolidationLlmMaxConcurrentDefault();
|
|
15819
15838
|
HINDSIGHT_PERF_DEFAULTS_UNGATED = [
|
|
15820
15839
|
[
|
|
15821
15840
|
"HINDSIGHT_API_RECALL_MAX_CANDIDATES_PER_SOURCE",
|
|
@@ -15869,6 +15888,14 @@ var init_hindsight_perf_defaults = __esm(() => {
|
|
|
15869
15888
|
[
|
|
15870
15889
|
"HINDSIGHT_API_CONSOLIDATION_MAX_MEMORIES_PER_ROUND",
|
|
15871
15890
|
String(HINDSIGHT_DEFAULT_CONSOLIDATION_MAX_MEMORIES_PER_ROUND)
|
|
15891
|
+
],
|
|
15892
|
+
[
|
|
15893
|
+
"HINDSIGHT_API_RECENCY_DECAY_FUNCTION",
|
|
15894
|
+
HINDSIGHT_DEFAULT_RECENCY_DECAY_FUNCTION
|
|
15895
|
+
],
|
|
15896
|
+
[
|
|
15897
|
+
"HINDSIGHT_API_RECENCY_DECAY_HALFLIFE_DAYS",
|
|
15898
|
+
String(HINDSIGHT_DEFAULT_RECENCY_DECAY_HALFLIFE_DAYS)
|
|
15872
15899
|
]
|
|
15873
15900
|
];
|
|
15874
15901
|
HINDSIGHT_PERF_DEFAULTS_GPU = [
|
|
@@ -15893,7 +15920,9 @@ var init_hindsight_perf_defaults = __esm(() => {
|
|
|
15893
15920
|
];
|
|
15894
15921
|
HINDSIGHT_PERF_OVERRIDE_ONLY_KEYS = new Set([
|
|
15895
15922
|
"HINDSIGHT_API_WORKER_CONSOLIDATION_BANK_PRIORITY",
|
|
15896
|
-
"HINDSIGHT_CE_DECISIVE_RELATIVE_GAP"
|
|
15923
|
+
"HINDSIGHT_CE_DECISIVE_RELATIVE_GAP",
|
|
15924
|
+
"HINDSIGHT_API_RECENCY_DECAY_LINEAR_WINDOW_DAYS",
|
|
15925
|
+
"HINDSIGHT_API_WORKER_MAX_SLOTS"
|
|
15897
15926
|
]);
|
|
15898
15927
|
HINDSIGHT_PERF_ENV_KEYS = new Set([
|
|
15899
15928
|
...[
|
|
@@ -16983,22 +17012,35 @@ function isHindsightEnabled(config) {
|
|
|
16983
17012
|
return false;
|
|
16984
17013
|
return config?.memory?.backend === "hindsight";
|
|
16985
17014
|
}
|
|
16986
|
-
function isUpgradableRetainMission(current) {
|
|
16987
|
-
if (current == null || current.trim() === "")
|
|
16988
|
-
return true;
|
|
16989
|
-
return SUPERSEDED_RETAIN_MISSIONS.includes(current);
|
|
16990
|
-
}
|
|
16991
17015
|
function decideRetainMissionUpgrade(configured, current) {
|
|
17016
|
+
return decideMissionUpgrade(configured, current, DEFAULT_RETAIN_MISSION, SUPERSEDED_RETAIN_MISSIONS);
|
|
17017
|
+
}
|
|
17018
|
+
function decideMissionUpgrade(configured, current, desired, shipped) {
|
|
16992
17019
|
if (configured)
|
|
16993
17020
|
return { action: "config", mission: configured };
|
|
16994
|
-
if (current ===
|
|
17021
|
+
if (current === desired)
|
|
16995
17022
|
return { action: "none" };
|
|
16996
|
-
if (
|
|
16997
|
-
return { action: "upgrade", mission:
|
|
17023
|
+
if (current == null || current.trim() === "" || shipped.includes(current)) {
|
|
17024
|
+
return { action: "upgrade", mission: desired };
|
|
16998
17025
|
}
|
|
16999
17026
|
return { action: "none" };
|
|
17000
17027
|
}
|
|
17028
|
+
function decideObservationsMissionUpgrade(configured, profileDefault, current) {
|
|
17029
|
+
const desired = profileDefault ?? DEFAULT_OBSERVATIONS_MISSION;
|
|
17030
|
+
const shipped = [
|
|
17031
|
+
...SUPERSEDED_OBSERVATIONS_MISSIONS,
|
|
17032
|
+
DEFAULT_OBSERVATIONS_MISSION,
|
|
17033
|
+
...Object.values(PROFILE_MEMORY_DEFAULTS).map((d) => d.observations_mission).filter((m) => m != null)
|
|
17034
|
+
].filter((m) => m !== desired);
|
|
17035
|
+
return decideMissionUpgrade(configured, current, desired, shipped);
|
|
17036
|
+
}
|
|
17001
17037
|
async function fetchBankRetainMission(apiUrl, bankId, opts) {
|
|
17038
|
+
return fetchBankMissionField(apiUrl, bankId, "retain_mission", opts);
|
|
17039
|
+
}
|
|
17040
|
+
async function fetchBankObservationsMission(apiUrl, bankId, opts) {
|
|
17041
|
+
return fetchBankMissionField(apiUrl, bankId, "observations_mission", opts);
|
|
17042
|
+
}
|
|
17043
|
+
async function fetchBankMissionField(apiUrl, bankId, field, opts) {
|
|
17002
17044
|
const fetchImpl = opts?.fetchImpl ?? fetch;
|
|
17003
17045
|
const timeoutMs = opts?.timeoutMs ?? 5000;
|
|
17004
17046
|
const base = apiUrl.replace(/\/mcp\/?$/, "").replace(/\/$/, "");
|
|
@@ -17015,10 +17057,10 @@ async function fetchBankRetainMission(apiUrl, bankId, opts) {
|
|
|
17015
17057
|
if (config == null || typeof config !== "object") {
|
|
17016
17058
|
return { ok: false, reason: "Unexpected shape" };
|
|
17017
17059
|
}
|
|
17018
|
-
if (!(
|
|
17060
|
+
if (!(field in config)) {
|
|
17019
17061
|
return { ok: false, reason: "Unexpected shape" };
|
|
17020
17062
|
}
|
|
17021
|
-
const mission = config
|
|
17063
|
+
const mission = config[field];
|
|
17022
17064
|
if (mission != null && typeof mission !== "string") {
|
|
17023
17065
|
return { ok: false, reason: "Unexpected shape" };
|
|
17024
17066
|
}
|
|
@@ -17570,7 +17612,7 @@ async function addMemoryTag(apiUrl, bankId, memoryId, tag, opts) {
|
|
|
17570
17612
|
return { ok: false, reason: String(err) };
|
|
17571
17613
|
}
|
|
17572
17614
|
}
|
|
17573
|
-
var HINDSIGHT_SHIM_CLI_PATH = "/usr/local/bin/switchroom", HINDSIGHT_SHIM_AGENT_HOME = "/state/agent/home", DEFAULT_RETAIN_MISSION, SUPERSEDED_RETAIN_MISSIONS, PROFILE_MEMORY_DEFAULTS, USER_PROFILE_SOURCE_QUERY = "What are the key facts, preferences, context, and communication style about the user I talk to? Summarize what matters for making the agent feel like it knows them.", DEMOTE_FROM_RECALL_TAG = "[demote-from-recall]";
|
|
17615
|
+
var HINDSIGHT_SHIM_CLI_PATH = "/usr/local/bin/switchroom", HINDSIGHT_SHIM_AGENT_HOME = "/state/agent/home", DEFAULT_RETAIN_MISSION, SUPERSEDED_RETAIN_MISSIONS, DEFAULT_OBSERVATIONS_MISSION, SUPERSEDED_OBSERVATIONS_MISSIONS, PROFILE_MEMORY_DEFAULTS, USER_PROFILE_SOURCE_QUERY = "What are the key facts, preferences, context, and communication style about the user I talk to? Summarize what matters for making the agent feel like it knows them.", DEMOTE_FROM_RECALL_TAG = "[demote-from-recall]";
|
|
17574
17616
|
var init_hindsight2 = __esm(() => {
|
|
17575
17617
|
init_users();
|
|
17576
17618
|
init_hindsight();
|
|
@@ -17599,6 +17641,16 @@ var init_hindsight2 = __esm(() => {
|
|
|
17599
17641
|
` + `- Hindsight's own errors, retries, backlogs, or internal state \u2014 the memory
|
|
17600
17642
|
` + ` system's self-reports are not memories.
|
|
17601
17643
|
` + `- Restatements of the user's current request or the task in progress.
|
|
17644
|
+
` + `- Volatile state written as a timeless assertion. A version, count, size,
|
|
17645
|
+
` + ` backlog, status, or any "X is running Y" / "X is at Y" / "X is currently Y"
|
|
17646
|
+
` + ` claim is true only at the instant it was said. Concretely, never produce a
|
|
17647
|
+
` + ` fact whose text resembles any of these: "Switchroom fleet is running image
|
|
17648
|
+
` + ` version v0.18.19", "The switchroom repo is at /path/to/fleet, version
|
|
17649
|
+
` + ` v0.19.5", "Bank overlord has 43155 pending consolidations", "The build is
|
|
17650
|
+
` + ` currently green". If the claim is worth keeping, put the date INSIDE the
|
|
17651
|
+
` + ` fact text ("As of 2026-07-19 the fleet was running v0.18.19"); if you
|
|
17652
|
+
` + ` cannot date it, drop it. An undated one is recalled forever as though it
|
|
17653
|
+
` + ` were still true, which is worse than not remembering it at all.
|
|
17602
17654
|
` + `- Transient state (unread counts, build status, what is running right now) unless
|
|
17603
17655
|
` + ` the fact is explicitly dated, in which case record it as a dated observation.
|
|
17604
17656
|
` + `- Greetings, acknowledgements, and routine operational chatter.
|
|
@@ -17620,18 +17672,108 @@ var init_hindsight2 = __esm(() => {
|
|
|
17620
17672
|
` + "- Transient state (unread counts, build status, what is running right now) " + "unless the fact is explicitly dated, in which case record it as a dated " + `observation.
|
|
17621
17673
|
` + `- Greetings, acknowledgements, and routine operational chatter.
|
|
17622
17674
|
|
|
17623
|
-
` + "If a candidate fact matches an exclusion, drop it rather than rewording " + "it. If nothing durable remains, return an empty facts list."
|
|
17675
|
+
` + "If a candidate fact matches an exclusion, drop it rather than rewording " + "it. If nothing durable remains, return an empty facts list.",
|
|
17676
|
+
`Extract durable facts that will still be true and useful weeks from now: user preferences and standing rules, ongoing projects and recurring commitments, technical and architectural decisions with their rationale, and people/tool relationships. A preference revealed by a request is durable \u2014 record the preference (what the user likes, wants, or always does), not the request itself.
|
|
17677
|
+
` + `
|
|
17678
|
+
` + `A TOOL RESULT IS NOT A FACT. Before extracting, ask: is the subject of this
|
|
17679
|
+
` + `candidate a file path, a command/process/agent/session id, a temp directory, or
|
|
17680
|
+
` + `the location where some output was written? If yes, drop it \u2014 it is transcript
|
|
17681
|
+
` + `exhaust, not memory.
|
|
17682
|
+
` + `
|
|
17683
|
+
` + `NEVER extract:
|
|
17684
|
+
` + `- Tool results verbatim or paraphrased. Concretely, never produce a fact whose
|
|
17685
|
+
` + ` text resembles any of these: "File created successfully at /path/to/file",
|
|
17686
|
+
` + ` "A background command with ID bctz4yskm is running, and its output will be
|
|
17687
|
+
` + ` written to /tmp/...", "Async agent a745598ba84e71df1 was launched successfully
|
|
17688
|
+
` + ` and is running in the background", "User executed a Bash command to sleep for
|
|
17689
|
+
` + ` 200 seconds", "The assistant used grep to locate 'truncateSync' in src/foo.ts".
|
|
17690
|
+
` + `- Anything mentioning a path under /tmp, a scratchpad directory, or a .tmp file.
|
|
17691
|
+
` + `- Agent tool-use traces or narration of what the assistant did (e.g. "the
|
|
17692
|
+
` + ` assistant used X to query Y", "ran a search", "sent the message").
|
|
17693
|
+
` + `- In-flight workflow/process narration (a sub-task started, paused, or is still
|
|
17694
|
+
` + ` running) \u2014 retain the outcome only once the task completes or a decision is made.
|
|
17695
|
+
` + `- Operation, request, batch, agent, command or session IDs, UUIDs, hashes, or error codes.
|
|
17696
|
+
` + `- Slash commands the user typed and their effects (e.g. "User issued /clear to
|
|
17697
|
+
` + ` reset assistant state").
|
|
17698
|
+
` + `- Hindsight's own errors, retries, backlogs, or internal state \u2014 the memory
|
|
17699
|
+
` + ` system's self-reports are not memories.
|
|
17700
|
+
` + `- Restatements of the user's current request or the task in progress.
|
|
17701
|
+
` + `- Transient state (unread counts, build status, what is running right now) unless
|
|
17702
|
+
` + ` the fact is explicitly dated, in which case record it as a dated observation.
|
|
17703
|
+
` + `- Greetings, acknowledgements, and routine operational chatter.
|
|
17704
|
+
` + `
|
|
17705
|
+
` + `If a candidate fact matches an exclusion, drop it rather than rewording it. If
|
|
17706
|
+
` + "nothing durable remains, return an empty facts list."
|
|
17707
|
+
];
|
|
17708
|
+
DEFAULT_OBSERVATIONS_MISSION = `Synthesise durable, standing knowledge about the people, projects, and systems this agent works with: preferences and standing rules, roles and relationships, skills and recurring patterns, technical and operational decisions with their rationale, and the state of long-running work once it lands.
|
|
17709
|
+
` + `
|
|
17710
|
+
` + `The test is durability, not notability: an observation must still be worth reading weeks from now. A single dated event belongs in an observation only when it establishes or changes a standing fact.
|
|
17711
|
+
` + `
|
|
17712
|
+
` + `Do NOT synthesise observations from:
|
|
17713
|
+
` + `- Transcript exhaust \u2014 tool calls and their results, file paths, temp or scratchpad directories, where some output was written, or narration of what an assistant did.
|
|
17714
|
+
` + `- Identifiers with no standing meaning: session, agent, request, batch or command IDs, UUIDs, hashes, error codes.
|
|
17715
|
+
` + `- In-flight process narration \u2014 a task started, paused, or still running. Record the outcome once it lands, not the running state.
|
|
17716
|
+
` + `- The memory system's own errors, retries, backlogs, or internal state.
|
|
17717
|
+
` + `- Transient state (what is running right now, unread counts, build status) unless the fact is explicitly dated, in which case record it as dated.
|
|
17718
|
+
` + `
|
|
17719
|
+
` + "If the new facts contain nothing durable, record nothing rather than synthesising a weak observation.";
|
|
17720
|
+
SUPERSEDED_OBSERVATIONS_MISSIONS = [
|
|
17721
|
+
"Synthesise the person's wellbeing patterns, motivations, and emotional " + "context \u2014 how habits, setbacks, and encouragement connect over time."
|
|
17624
17722
|
];
|
|
17625
17723
|
PROFILE_MEMORY_DEFAULTS = {
|
|
17626
17724
|
"health-coach": {
|
|
17627
17725
|
disposition: { skepticism: 2, literalism: 2, empathy: 5 },
|
|
17628
|
-
observations_mission:
|
|
17726
|
+
observations_mission: `You consolidate the memory of a health and fitness coach working with one person. This bank records how that person actually lives and trains.
|
|
17727
|
+
` + `
|
|
17728
|
+
` + `Synthesise into durable observations:
|
|
17729
|
+
` + `- Goals, targets, and the plan currently in force, with the reasoning behind each.
|
|
17730
|
+
` + `- Training, nutrition, sleep, and alcohol patterns as they hold over weeks \u2014 what the person reliably does, not what they did once.
|
|
17731
|
+
` + `- Constraints that shape the plan: injuries, medical guidance, schedule, equipment, foods and sessions they refuse.
|
|
17732
|
+
` + `- Motivations, and what actually helps or backfires when they slip.
|
|
17733
|
+
` + `- Trends in the numbers: direction and range over time, not any single reading.
|
|
17734
|
+
` + `- Corrections to earlier beliefs, recorded explicitly as corrections.
|
|
17735
|
+
` + `
|
|
17736
|
+
` + `A single day's log, weight reading, or session is evidence, not an observation. Record one only when it establishes or changes a standing pattern, target, or constraint \u2014 then fold it into the observation for that pattern and say what changed and roughly when.
|
|
17737
|
+
` + `
|
|
17738
|
+
` + `Granularity: one observation per habit, target, constraint, or trend. Aggregate repeated daily evidence into the observation for that pattern rather than creating one per day, and keep training, nutrition, and sleep as separate observations.
|
|
17739
|
+
` + `
|
|
17740
|
+
` + "Word observations as the person's own pattern and framing, never as a verdict on them."
|
|
17629
17741
|
},
|
|
17630
17742
|
"executive-assistant": {
|
|
17631
|
-
disposition: { skepticism: 4, literalism: 4, empathy: 3 }
|
|
17743
|
+
disposition: { skepticism: 4, literalism: 4, empathy: 3 },
|
|
17744
|
+
observations_mission: `You consolidate the memory of an executive assistant working for one person. This bank records that person's commitments, people, and standing arrangements.
|
|
17745
|
+
` + `
|
|
17746
|
+
` + `Synthesise into durable observations:
|
|
17747
|
+
` + `- Standing rules and preferences: how they want things scheduled, written, and filed, and when they want to be interrupted.
|
|
17748
|
+
` + `- People and organisations, and the relationship: who they are, what they are involved in, how to reach them.
|
|
17749
|
+
` + `- Recurring commitments and routines, and the constraints around them.
|
|
17750
|
+
` + `- Obligations and their state: what was promised to whom, the deadline, and what is still outstanding.
|
|
17751
|
+
` + `- Decisions made and decisions deferred, with the reasoning and the trade accepted.
|
|
17752
|
+
` + `- Corrections to earlier beliefs, recorded explicitly as corrections.
|
|
17753
|
+
` + `
|
|
17754
|
+
` + `Live commitment state IS durable knowledge here, not ephemeral chatter. An outstanding obligation, a travel window, an unanswered request, a "currently X" arrangement \u2014 these are precisely what this agent must recall later. Keep them, and embed the date inside the observation text so a later reader can judge staleness. Do not drop them as ephemeral.
|
|
17755
|
+
` + `
|
|
17756
|
+
` + `Granularity: one observation per person, arrangement, or obligation. Aggregate repeated mentions into that observation rather than creating siblings; never merge two different people or two different commitments into one.
|
|
17757
|
+
` + `
|
|
17758
|
+
` + "Do not synthesise from transcript exhaust: tool calls and their results, message or event identifiers, or narration of what the assistant did."
|
|
17632
17759
|
},
|
|
17633
17760
|
coding: {
|
|
17634
|
-
disposition: { skepticism: 4, literalism: 5, empathy: 2 }
|
|
17761
|
+
disposition: { skepticism: 4, literalism: 5, empathy: 2 },
|
|
17762
|
+
observations_mission: `You consolidate the memory of a software-engineering agent. This bank records real work on real codebases.
|
|
17763
|
+
` + `
|
|
17764
|
+
` + `Synthesise into durable observations:
|
|
17765
|
+
` + `- Architecture and design decisions, each with its rationale and the trade accepted.
|
|
17766
|
+
` + `- Root causes, with the evidence chain, and negative results \u2014 what was ruled out matters as much as what was found.
|
|
17767
|
+
` + `- How the repository works: build, test, and lint commands, conventions, CI gates, and where things live.
|
|
17768
|
+
` + `- Outcomes of code work: issue and PR numbers, what changed, whether it merged, what review found.
|
|
17769
|
+
` + `- The user's standing rules, preferences, and corrections to this agent's behaviour.
|
|
17770
|
+
` + `- Corrections to earlier beliefs, recorded explicitly as corrections.
|
|
17771
|
+
` + `
|
|
17772
|
+
` + `Repository and service state IS durable knowledge here, not ephemeral chatter. Versions, a failing gate, an open PR, a measured number, a "currently X" claim \u2014 these are precisely what this agent must recall later. Keep them, and embed the date inside the observation text so a later reader can judge staleness. Do not drop them as ephemeral.
|
|
17773
|
+
` + `
|
|
17774
|
+
` + `Granularity: one observation per distinct decision, cause, convention, or work item. Aggregate repeated evidence about the same one into that observation rather than creating siblings; never merge two separate decisions into a single summary.
|
|
17775
|
+
` + `
|
|
17776
|
+
` + "Do not synthesise from transcript exhaust: tool calls and their results, scratch paths, session or request identifiers, or narration of what the agent did. Prefer specific and falsifiable \u2014 naming the file, the number, the commit, or the decision beats summarising the topic."
|
|
17635
17777
|
}
|
|
17636
17778
|
};
|
|
17637
17779
|
});
|
|
@@ -27998,6 +28140,21 @@ async function resolveRetainMissionPush(apiUrl, bankId, configured, label) {
|
|
|
27998
28140
|
}
|
|
27999
28141
|
return decision.mission;
|
|
28000
28142
|
}
|
|
28143
|
+
async function resolveObservationsMissionPush(apiUrl, bankId, configured, profileName, label) {
|
|
28144
|
+
if (configured)
|
|
28145
|
+
return configured;
|
|
28146
|
+
const profileDefault = PROFILE_MEMORY_DEFAULTS[profileName]?.observations_mission;
|
|
28147
|
+
const current = await fetchBankObservationsMission(apiUrl, bankId, { timeoutMs: 5000 });
|
|
28148
|
+
if (!current.ok) {
|
|
28149
|
+
console.warn(` ${source_default.yellow("\u26a0")} Could not read current observations_mission for ${label} (${current.reason}) \u2014 leaving it untouched`);
|
|
28150
|
+
return;
|
|
28151
|
+
}
|
|
28152
|
+
const decision = decideObservationsMissionUpgrade(configured, profileDefault, current.mission);
|
|
28153
|
+
if (decision.action === "upgrade") {
|
|
28154
|
+
console.log(` ${source_default.green("\u2713")} Seeding default observations_mission for ${label}`);
|
|
28155
|
+
}
|
|
28156
|
+
return decision.mission;
|
|
28157
|
+
}
|
|
28001
28158
|
function parseDurationToSeconds(d) {
|
|
28002
28159
|
if (!d)
|
|
28003
28160
|
return;
|
|
@@ -29504,6 +29661,11 @@ ${body}
|
|
|
29504
29661
|
if (seededRetainMission) {
|
|
29505
29662
|
missions.retain_mission = seededRetainMission;
|
|
29506
29663
|
}
|
|
29664
|
+
delete missions.observations_mission;
|
|
29665
|
+
const seededObservationsMission = await resolveObservationsMissionPush(apiUrl, hindsightBankId, agentConfig.memory?.observations_mission, agentConfig.extends ?? DEFAULT_PROFILE, formatAgentBankLabel(name, hindsightBankId));
|
|
29666
|
+
if (seededObservationsMission) {
|
|
29667
|
+
missions.observations_mission = seededObservationsMission;
|
|
29668
|
+
}
|
|
29507
29669
|
if (userBankMission) {
|
|
29508
29670
|
missions.bank_mission = userBankMission;
|
|
29509
29671
|
}
|
|
@@ -30574,6 +30736,10 @@ ${body}
|
|
|
30574
30736
|
const retainMission = await resolveRetainMissionPush(apiUrl, hindsightBankId, agentConfig.memory?.retain_mission, formatAgentBankLabel(name, hindsightBankId));
|
|
30575
30737
|
if (retainMission)
|
|
30576
30738
|
missions.retain_mission = retainMission;
|
|
30739
|
+
delete missions.observations_mission;
|
|
30740
|
+
const observationsMission = await resolveObservationsMissionPush(apiUrl, hindsightBankId, agentConfig.memory?.observations_mission, agentConfig.extends ?? DEFAULT_PROFILE, formatAgentBankLabel(name, hindsightBankId));
|
|
30741
|
+
if (observationsMission)
|
|
30742
|
+
missions.observations_mission = observationsMission;
|
|
30577
30743
|
if (Object.keys(missions).length > 0) {
|
|
30578
30744
|
await updateBankMissions(apiUrl, hindsightBankId, missions, { timeoutMs: 5000 }).then((result) => {
|
|
30579
30745
|
if (result.ok) {
|
|
@@ -45612,6 +45778,7 @@ var exports_provision = {};
|
|
|
45612
45778
|
__export(exports_provision, {
|
|
45613
45779
|
validateKey: () => validateKey,
|
|
45614
45780
|
updateKeyModels: () => updateKeyModels,
|
|
45781
|
+
updateKeyBudget: () => updateKeyBudget,
|
|
45615
45782
|
ensureTeam: () => ensureTeam,
|
|
45616
45783
|
ensureKey: () => ensureKey,
|
|
45617
45784
|
bindKeyToTeam: () => bindKeyToTeam,
|
|
@@ -45733,6 +45900,12 @@ async function generateKeyOnce(opts, fetchFn) {
|
|
|
45733
45900
|
if (opts.metadata && Object.keys(opts.metadata).length > 0) {
|
|
45734
45901
|
payload.metadata = opts.metadata;
|
|
45735
45902
|
}
|
|
45903
|
+
if (opts.budget) {
|
|
45904
|
+
payload.max_budget = opts.budget.maxBudget;
|
|
45905
|
+
payload.budget_duration = opts.budget.budgetDuration;
|
|
45906
|
+
if (opts.budget.softBudget != null)
|
|
45907
|
+
payload.soft_budget = opts.budget.softBudget;
|
|
45908
|
+
}
|
|
45736
45909
|
let resp;
|
|
45737
45910
|
try {
|
|
45738
45911
|
resp = await fetchFn(url, {
|
|
@@ -45918,6 +46091,27 @@ async function safeText(resp) {
|
|
|
45918
46091
|
return "";
|
|
45919
46092
|
}
|
|
45920
46093
|
}
|
|
46094
|
+
async function updateKeyBudget(opts, fetchFn = fetch) {
|
|
46095
|
+
const url = `${normalizeBase(opts.baseUrl)}/key/update`;
|
|
46096
|
+
let resp;
|
|
46097
|
+
try {
|
|
46098
|
+
resp = await fetchFn(url, {
|
|
46099
|
+
method: "POST",
|
|
46100
|
+
headers: authHeaders(opts.masterKey),
|
|
46101
|
+
body: JSON.stringify({
|
|
46102
|
+
key: opts.key,
|
|
46103
|
+
max_budget: opts.budget.maxBudget,
|
|
46104
|
+
budget_duration: opts.budget.budgetDuration
|
|
46105
|
+
})
|
|
46106
|
+
});
|
|
46107
|
+
} catch (err) {
|
|
46108
|
+
return { kind: "error", detail: err.message };
|
|
46109
|
+
}
|
|
46110
|
+
if (resp.ok)
|
|
46111
|
+
return { kind: "ok" };
|
|
46112
|
+
const body = await safeText(resp);
|
|
46113
|
+
return { kind: "error", detail: `HTTP ${resp.status}: ${body.slice(0, 200)}` };
|
|
46114
|
+
}
|
|
45921
46115
|
var LiteLLMProvisionError;
|
|
45922
46116
|
var init_provision = __esm(() => {
|
|
45923
46117
|
LiteLLMProvisionError = class LiteLLMProvisionError extends Error {
|
|
@@ -47312,6 +47506,176 @@ var init_doctor_webkite = __esm(() => {
|
|
|
47312
47506
|
init_loader();
|
|
47313
47507
|
});
|
|
47314
47508
|
|
|
47509
|
+
// src/openrouter/credit.ts
|
|
47510
|
+
function num(v) {
|
|
47511
|
+
return typeof v === "number" && Number.isFinite(v) ? v : null;
|
|
47512
|
+
}
|
|
47513
|
+
function parseKeySnapshot(body) {
|
|
47514
|
+
if (body == null || typeof body !== "object")
|
|
47515
|
+
return null;
|
|
47516
|
+
const data = body.data;
|
|
47517
|
+
if (data == null || typeof data !== "object")
|
|
47518
|
+
return null;
|
|
47519
|
+
const d = data;
|
|
47520
|
+
if (num(d.usage) == null)
|
|
47521
|
+
return null;
|
|
47522
|
+
return {
|
|
47523
|
+
label: typeof d.label === "string" ? d.label : null,
|
|
47524
|
+
limit: num(d.limit),
|
|
47525
|
+
limitRemaining: num(d.limit_remaining),
|
|
47526
|
+
limitReset: typeof d.limit_reset === "string" ? d.limit_reset : null,
|
|
47527
|
+
usage: num(d.usage),
|
|
47528
|
+
isFreeTier: typeof d.is_free_tier === "boolean" ? d.is_free_tier : null
|
|
47529
|
+
};
|
|
47530
|
+
}
|
|
47531
|
+
function evaluateCredit(snapshot) {
|
|
47532
|
+
const usage = snapshot.usage ?? 0;
|
|
47533
|
+
if (snapshot.limit == null || snapshot.limitRemaining == null) {
|
|
47534
|
+
return {
|
|
47535
|
+
status: "warn",
|
|
47536
|
+
detail: `OpenRouter key has NO per-key credit limit set, so its remaining credit cannot be measured. ` + `GET /api/v1/key reports limit/limit_remaining as null when a key is uncapped, and the response ` + `carries no account-balance field \u2014 so there is nothing to threshold and this check CANNOT warn ` + `you before the account runs dry. (Key has spent $${usage.toFixed(2)} all-time.)`,
|
|
47537
|
+
fix: `Set a per-key credit limit at ${OPENROUTER_KEYS_URL} (vault key: \`${OPENROUTER_VAULT_KEY}\`). ` + `That is what makes proactive monitoring possible at all, and it bounds the damage if the proxy ` + `key ever leaks. This is a manual console action \u2014 OpenRouter exposes no API to set it.`,
|
|
47538
|
+
actionable: false
|
|
47539
|
+
};
|
|
47540
|
+
}
|
|
47541
|
+
const { limit, limitRemaining } = snapshot;
|
|
47542
|
+
const fraction = limit > 0 ? limitRemaining / limit : 0;
|
|
47543
|
+
const reset = snapshot.limitReset ? ` (resets: ${snapshot.limitReset})` : " (never resets)";
|
|
47544
|
+
const where = `$${limitRemaining.toFixed(2)} of $${limit.toFixed(2)} remaining${reset}`;
|
|
47545
|
+
if (limitRemaining <= 0) {
|
|
47546
|
+
return {
|
|
47547
|
+
status: "fail",
|
|
47548
|
+
detail: `OpenRouter key credit is EXHAUSTED \u2014 ${where}. Every request through it is answering HTTP 402.`,
|
|
47549
|
+
fix: `Top up at ${OPENROUTER_CREDITS_URL} or raise the key's limit at ${OPENROUTER_KEYS_URL} (vault key: \`${OPENROUTER_VAULT_KEY}\`).`,
|
|
47550
|
+
actionable: true
|
|
47551
|
+
};
|
|
47552
|
+
}
|
|
47553
|
+
if (fraction < FAIL_REMAINING_FRACTION || limitRemaining < FAIL_REMAINING_USD) {
|
|
47554
|
+
return {
|
|
47555
|
+
status: "fail",
|
|
47556
|
+
detail: `OpenRouter key credit is CRITICALLY low \u2014 ${where}. A single busy session can exhaust it.`,
|
|
47557
|
+
fix: `Top up at ${OPENROUTER_CREDITS_URL} or raise the key's limit at ${OPENROUTER_KEYS_URL} (vault key: \`${OPENROUTER_VAULT_KEY}\`).`,
|
|
47558
|
+
actionable: true
|
|
47559
|
+
};
|
|
47560
|
+
}
|
|
47561
|
+
if (fraction < WARN_REMAINING_FRACTION || limitRemaining < WARN_REMAINING_USD) {
|
|
47562
|
+
return {
|
|
47563
|
+
status: "warn",
|
|
47564
|
+
detail: `OpenRouter key credit is running low \u2014 ${where}.`,
|
|
47565
|
+
fix: `Top up at ${OPENROUTER_CREDITS_URL} before it reaches zero (vault key: \`${OPENROUTER_VAULT_KEY}\`).`,
|
|
47566
|
+
actionable: true
|
|
47567
|
+
};
|
|
47568
|
+
}
|
|
47569
|
+
return {
|
|
47570
|
+
status: "ok",
|
|
47571
|
+
detail: `OpenRouter key credit healthy \u2014 ${where}.`,
|
|
47572
|
+
actionable: false
|
|
47573
|
+
};
|
|
47574
|
+
}
|
|
47575
|
+
async function fetchKeySnapshot(apiKey, fetchFn, signal) {
|
|
47576
|
+
let resp;
|
|
47577
|
+
try {
|
|
47578
|
+
resp = await fetchFn(OPENROUTER_KEY_ENDPOINT, {
|
|
47579
|
+
headers: { authorization: `Bearer ${apiKey}` },
|
|
47580
|
+
signal
|
|
47581
|
+
});
|
|
47582
|
+
} catch (err) {
|
|
47583
|
+
return { kind: "unreachable", detail: err.message };
|
|
47584
|
+
}
|
|
47585
|
+
if (resp.status === 401 || resp.status === 403) {
|
|
47586
|
+
return { kind: "unauthorized", status: resp.status };
|
|
47587
|
+
}
|
|
47588
|
+
if (!resp.ok)
|
|
47589
|
+
return { kind: "unreachable", detail: `HTTP ${resp.status}` };
|
|
47590
|
+
let body;
|
|
47591
|
+
try {
|
|
47592
|
+
body = await resp.json();
|
|
47593
|
+
} catch {
|
|
47594
|
+
return { kind: "unexpected-shape" };
|
|
47595
|
+
}
|
|
47596
|
+
const snapshot = parseKeySnapshot(body);
|
|
47597
|
+
if (snapshot == null)
|
|
47598
|
+
return { kind: "unexpected-shape" };
|
|
47599
|
+
return { kind: "ok", snapshot };
|
|
47600
|
+
}
|
|
47601
|
+
function assessFetchFailure(result) {
|
|
47602
|
+
switch (result.kind) {
|
|
47603
|
+
case "unauthorized":
|
|
47604
|
+
return {
|
|
47605
|
+
status: "fail",
|
|
47606
|
+
detail: `OpenRouter rejected the credit query with HTTP ${result.status} \u2014 the key is invalid, revoked or disabled.`,
|
|
47607
|
+
fix: `Re-issue the key at ${OPENROUTER_KEYS_URL} and update the vault entry \`${OPENROUTER_VAULT_KEY}\`.`,
|
|
47608
|
+
actionable: true
|
|
47609
|
+
};
|
|
47610
|
+
case "unexpected-shape":
|
|
47611
|
+
return {
|
|
47612
|
+
status: "warn",
|
|
47613
|
+
detail: `OpenRouter's GET /api/v1/key returned an unrecognised body, so credit could not be read. ` + `Treated as UNKNOWN, not as healthy \u2014 the API shape may have changed.`,
|
|
47614
|
+
fix: `Re-check https://openrouter.ai/docs/api-reference/limits against src/openrouter/credit.ts.`,
|
|
47615
|
+
actionable: false
|
|
47616
|
+
};
|
|
47617
|
+
case "unreachable":
|
|
47618
|
+
return {
|
|
47619
|
+
status: "skip",
|
|
47620
|
+
detail: `Could not reach OpenRouter to check credit (${result.detail}).`,
|
|
47621
|
+
actionable: false
|
|
47622
|
+
};
|
|
47623
|
+
}
|
|
47624
|
+
}
|
|
47625
|
+
var WARN_REMAINING_FRACTION = 0.25, WARN_REMAINING_USD = 10, FAIL_REMAINING_FRACTION = 0.1, FAIL_REMAINING_USD = 3, OPENROUTER_VAULT_KEY = "openrouter/api-key", OPENROUTER_KEYS_URL = "https://openrouter.ai/settings/keys", OPENROUTER_CREDITS_URL = "https://openrouter.ai/credits", OPENROUTER_KEY_ENDPOINT = "https://openrouter.ai/api/v1/key";
|
|
47626
|
+
|
|
47627
|
+
// src/cli/doctor-openrouter-credit.ts
|
|
47628
|
+
async function runOpenRouterCreditChecks(deps) {
|
|
47629
|
+
if (!deps.enabled) {
|
|
47630
|
+
return [
|
|
47631
|
+
{
|
|
47632
|
+
name: CHECK_NAME2,
|
|
47633
|
+
status: "skip",
|
|
47634
|
+
detail: "No OpenRouter-backed model is configured in the LiteLLM proxy config."
|
|
47635
|
+
}
|
|
47636
|
+
];
|
|
47637
|
+
}
|
|
47638
|
+
let apiKey;
|
|
47639
|
+
try {
|
|
47640
|
+
apiKey = await deps.readSecret(OPENROUTER_VAULT_KEY);
|
|
47641
|
+
} catch (err) {
|
|
47642
|
+
return [
|
|
47643
|
+
{
|
|
47644
|
+
name: CHECK_NAME2,
|
|
47645
|
+
status: "skip",
|
|
47646
|
+
detail: `Could not read vault key \`${OPENROUTER_VAULT_KEY}\` (${err.message}).`
|
|
47647
|
+
}
|
|
47648
|
+
];
|
|
47649
|
+
}
|
|
47650
|
+
if (!apiKey) {
|
|
47651
|
+
return [
|
|
47652
|
+
{
|
|
47653
|
+
name: CHECK_NAME2,
|
|
47654
|
+
status: "warn",
|
|
47655
|
+
detail: `The proxy routes through OpenRouter but vault key \`${OPENROUTER_VAULT_KEY}\` is not ` + `readable here, so credit cannot be monitored.`,
|
|
47656
|
+
fix: `Grant read access to \`${OPENROUTER_VAULT_KEY}\`, or store it if it is missing.`
|
|
47657
|
+
}
|
|
47658
|
+
];
|
|
47659
|
+
}
|
|
47660
|
+
const fetched = await fetchKeySnapshot(apiKey, deps.fetchFn);
|
|
47661
|
+
const assessment = fetched.kind === "ok" ? evaluateCredit(fetched.snapshot) : assessFetchFailure(fetched);
|
|
47662
|
+
return [
|
|
47663
|
+
{
|
|
47664
|
+
name: CHECK_NAME2,
|
|
47665
|
+
status: assessment.status,
|
|
47666
|
+
detail: assessment.detail,
|
|
47667
|
+
...assessment.fix ? { fix: assessment.fix } : {}
|
|
47668
|
+
}
|
|
47669
|
+
];
|
|
47670
|
+
}
|
|
47671
|
+
function usesOpenRouter(litellmConfigText) {
|
|
47672
|
+
if (!litellmConfigText)
|
|
47673
|
+
return false;
|
|
47674
|
+
return litellmConfigText.includes("openrouter/") || litellmConfigText.includes("OPENROUTER_API_KEY");
|
|
47675
|
+
}
|
|
47676
|
+
var CHECK_NAME2 = "OpenRouter credit";
|
|
47677
|
+
var init_doctor_openrouter_credit = () => {};
|
|
47678
|
+
|
|
47315
47679
|
// src/cli/doctor-cron-session.ts
|
|
47316
47680
|
import { statSync as realStatSync } from "node:fs";
|
|
47317
47681
|
import { resolve as resolve38 } from "node:path";
|
|
@@ -47388,18 +47752,18 @@ function parseEpisode(raw) {
|
|
|
47388
47752
|
if (typeof raw !== "object" || raw === null)
|
|
47389
47753
|
return null;
|
|
47390
47754
|
const r = raw;
|
|
47391
|
-
const
|
|
47392
|
-
const firstTs =
|
|
47393
|
-
const untilTs =
|
|
47394
|
-
const peak =
|
|
47755
|
+
const num2 = (v) => typeof v === "number" && Number.isFinite(v) ? v : null;
|
|
47756
|
+
const firstTs = num2(r.firstTs);
|
|
47757
|
+
const untilTs = num2(r.untilTs);
|
|
47758
|
+
const peak = num2(r.peakRetryAfterSec);
|
|
47395
47759
|
if (firstTs === null || untilTs === null || peak === null)
|
|
47396
47760
|
return null;
|
|
47397
47761
|
return {
|
|
47398
47762
|
firstTs,
|
|
47399
|
-
lastTs:
|
|
47763
|
+
lastTs: num2(r.lastTs) ?? firstTs,
|
|
47400
47764
|
peakRetryAfterSec: peak,
|
|
47401
47765
|
untilTs,
|
|
47402
|
-
count:
|
|
47766
|
+
count: num2(r.count) ?? 1
|
|
47403
47767
|
};
|
|
47404
47768
|
}
|
|
47405
47769
|
function readFlood429LedgerResult(path6, readFile2 = (p) => readFileSync63(p, "utf-8")) {
|
|
@@ -47571,6 +47935,25 @@ var init_doctor_flood_pressure = __esm(() => {
|
|
|
47571
47935
|
FIX2 = "A flood ban is server-side and cannot be cleared early \u2014 cut the outbound " + "rate that earns it. Check the gateway's `send-gate stats` and " + "`edit-flood-fuse` lines for the class doing the sending (progress cards and " + "the worker feed are the usual culprits), and see " + "`telegram-plugin/edit-flood-fuse.ts` for the per-class ceilings.";
|
|
47572
47936
|
});
|
|
47573
47937
|
|
|
47938
|
+
// src/litellm/budget.ts
|
|
47939
|
+
function resolveKeyBudget(config) {
|
|
47940
|
+
const raw = config?.max_budget;
|
|
47941
|
+
const maxBudget = raw === undefined ? DEFAULT_KEY_MAX_BUDGET_USD : raw;
|
|
47942
|
+
if (!Number.isFinite(maxBudget) || maxBudget <= 0)
|
|
47943
|
+
return null;
|
|
47944
|
+
const budgetDuration = config?.budget_duration && config.budget_duration.length > 0 ? config.budget_duration : DEFAULT_KEY_BUDGET_DURATION;
|
|
47945
|
+
const rawSoft = config?.soft_budget ?? DEFAULT_KEY_SOFT_BUDGET_USD;
|
|
47946
|
+
const softBudget = Number.isFinite(rawSoft) && rawSoft > 0 && rawSoft < maxBudget ? rawSoft : undefined;
|
|
47947
|
+
return { maxBudget, softBudget, budgetDuration };
|
|
47948
|
+
}
|
|
47949
|
+
function describeKeyBudget(budget) {
|
|
47950
|
+
if (budget == null)
|
|
47951
|
+
return "no spend cap (uncapped key)";
|
|
47952
|
+
const soft = budget.softBudget != null ? `, alert at $${budget.softBudget}` : "";
|
|
47953
|
+
return `cap $${budget.maxBudget}/${budget.budgetDuration}${soft}`;
|
|
47954
|
+
}
|
|
47955
|
+
var DEFAULT_KEY_MAX_BUDGET_USD = 25, DEFAULT_KEY_SOFT_BUDGET_USD = 15, DEFAULT_KEY_BUDGET_DURATION = "30d";
|
|
47956
|
+
|
|
47574
47957
|
// examples/switchroom.yaml
|
|
47575
47958
|
var switchroom_default = `# switchroom.yaml \u2014 Full example configuration
|
|
47576
47959
|
#
|
|
@@ -48725,7 +49108,7 @@ async function provisionLiteLLMKeys(config, agentNames, switchroomConfigPath, ct
|
|
|
48725
49108
|
let brokerKeyMaterialized = false;
|
|
48726
49109
|
const [
|
|
48727
49110
|
{ getViaBrokerStructured: getViaBrokerStructured2, putViaBroker: putViaBroker2 },
|
|
48728
|
-
{ ensureTeam: ensureTeam2, ensureKey: ensureKey2, validateKey: validateKey2, bindKeyToTeam: bindKeyToTeam2, updateKeyModels: updateKeyModels2 },
|
|
49111
|
+
{ ensureTeam: ensureTeam2, ensureKey: ensureKey2, validateKey: validateKey2, bindKeyToTeam: bindKeyToTeam2, updateKeyModels: updateKeyModels2, updateKeyBudget: updateKeyBudget2 },
|
|
48729
49112
|
{ addAgentSecret: addAgentSecret2 }
|
|
48730
49113
|
] = await Promise.all([
|
|
48731
49114
|
Promise.resolve().then(() => (init_client(), exports_client)),
|
|
@@ -48806,6 +49189,11 @@ async function provisionLiteLLMKeys(config, agentNames, switchroomConfigPath, ct
|
|
|
48806
49189
|
const baseUrl = litellm.base_url ?? topLevel?.base_url;
|
|
48807
49190
|
const team = litellm.team ?? topLevel?.team ?? "switchroom";
|
|
48808
49191
|
const adminKeyRef = litellm.admin_key ?? topLevel?.admin_key;
|
|
49192
|
+
const budget = resolveKeyBudget({
|
|
49193
|
+
max_budget: litellm.max_budget ?? topLevel?.max_budget,
|
|
49194
|
+
soft_budget: litellm.soft_budget ?? topLevel?.soft_budget,
|
|
49195
|
+
budget_duration: litellm.budget_duration ?? topLevel?.budget_duration
|
|
49196
|
+
});
|
|
48809
49197
|
if (!baseUrl || !adminKeyRef) {
|
|
48810
49198
|
failures.push({
|
|
48811
49199
|
agent: name,
|
|
@@ -48887,6 +49275,16 @@ async function provisionLiteLLMKeys(config, agentNames, switchroomConfigPath, ct
|
|
|
48887
49275
|
writeOut(source_default.gray(` ~ litellm/${name}: key already provisioned (skipped)
|
|
48888
49276
|
`));
|
|
48889
49277
|
}
|
|
49278
|
+
if (!driftReprovision && storedKey && masterKey && budget) {
|
|
49279
|
+
const cap = await updateKeyBudget2({ baseUrl, masterKey, key: storedKey, budget });
|
|
49280
|
+
if (cap.kind === "ok") {
|
|
49281
|
+
writeOut(source_default.gray(` ~ litellm/${name}: spend ${describeKeyBudget(budget)}${budget.softBudget != null ? " (soft budget applies to newly generated keys only)" : ""}
|
|
49282
|
+
`));
|
|
49283
|
+
} else {
|
|
49284
|
+
ctx.writeErr(source_default.yellow(` ! litellm/${name}: could NOT apply the spend cap (${cap.detail}) \u2014 ` + `this key remains UNCAPPED and can spend the whole upstream account balance. Manual: POST ${baseUrl}/key/update {"key":"<key>","max_budget":${budget.maxBudget},"budget_duration":"${budget.budgetDuration}"}.
|
|
49285
|
+
`));
|
|
49286
|
+
}
|
|
49287
|
+
}
|
|
48890
49288
|
if (!driftReprovision) {
|
|
48891
49289
|
if (configText !== null) {
|
|
48892
49290
|
const after = addAgentSecret2(configText, name, vaultKey);
|
|
@@ -48923,6 +49321,7 @@ async function provisionLiteLLMKeys(config, agentNames, switchroomConfigPath, ct
|
|
|
48923
49321
|
alias,
|
|
48924
49322
|
teamId,
|
|
48925
49323
|
metadata,
|
|
49324
|
+
...budget ? { budget } : {},
|
|
48926
49325
|
log: (m) => ctx.writeErr(source_default.gray(` ~ litellm/${name}: ${m}
|
|
48927
49326
|
`))
|
|
48928
49327
|
});
|
|
@@ -52352,6 +52751,7 @@ __export(exports_doctor, {
|
|
|
52352
52751
|
checkDepsCacheWritable: () => checkDepsCacheWritable,
|
|
52353
52752
|
checkDeployMounts: () => checkDeployMounts,
|
|
52354
52753
|
checkConfig: () => checkConfig,
|
|
52754
|
+
checkBankObservationsMissions: () => checkBankObservationsMissions,
|
|
52355
52755
|
checkBankIngestHealth: () => checkBankIngestHealth,
|
|
52356
52756
|
checkAgents: () => checkAgents,
|
|
52357
52757
|
buildPendingRetainsProbeScript: () => buildPendingRetainsProbeScript,
|
|
@@ -52372,6 +52772,20 @@ import {
|
|
|
52372
52772
|
} from "node:fs";
|
|
52373
52773
|
import { dirname as dirname31, join as join81, resolve as resolve44 } from "node:path";
|
|
52374
52774
|
import { createPublicKey, createPrivateKey } from "node:crypto";
|
|
52775
|
+
function readLitellmConfigText() {
|
|
52776
|
+
const candidates = [
|
|
52777
|
+
process.env.LITELLM_CONFIG_PATH,
|
|
52778
|
+
"/data/coolify/services/litellm/litellm-config.yaml",
|
|
52779
|
+
join81(process.cwd(), "docker", "litellm-proxy", "litellm-config.yaml")
|
|
52780
|
+
].filter((p) => typeof p === "string" && p.length > 0);
|
|
52781
|
+
for (const p of candidates) {
|
|
52782
|
+
try {
|
|
52783
|
+
if (existsSync77(p))
|
|
52784
|
+
return readFileSync71(p, "utf-8");
|
|
52785
|
+
} catch {}
|
|
52786
|
+
}
|
|
52787
|
+
return null;
|
|
52788
|
+
}
|
|
52375
52789
|
function findInNvm(bin) {
|
|
52376
52790
|
const nvmRoot = join81(process.env.HOME ?? "", ".nvm", "versions", "node");
|
|
52377
52791
|
if (!existsSync77(nvmRoot))
|
|
@@ -53041,6 +53455,58 @@ function probeAuthBrokerSocket(consumerName) {
|
|
|
53041
53455
|
return "unreachable";
|
|
53042
53456
|
return "missing";
|
|
53043
53457
|
}
|
|
53458
|
+
async function checkBankObservationsMissions(config, url, opts) {
|
|
53459
|
+
const banks = new Map;
|
|
53460
|
+
for (const [agentName, agentConfig] of Object.entries(config.agents)) {
|
|
53461
|
+
const resolved = resolveAgentConfig(config.defaults, config.profiles, agentConfig);
|
|
53462
|
+
const bankId = resolved.memory?.collection ?? agentName;
|
|
53463
|
+
const existing = banks.get(bankId);
|
|
53464
|
+
if (existing)
|
|
53465
|
+
existing.agents.push(agentName);
|
|
53466
|
+
else
|
|
53467
|
+
banks.set(bankId, { agents: [agentName], config: resolved });
|
|
53468
|
+
}
|
|
53469
|
+
const inspected = await Promise.all([...banks].map(async ([bankId, entry]) => [
|
|
53470
|
+
bankId,
|
|
53471
|
+
entry,
|
|
53472
|
+
await fetchBankObservationsMission(url, bankId, { fetchImpl: opts?.fetchImpl })
|
|
53473
|
+
]));
|
|
53474
|
+
return inspected.map(([bankId, entry, read]) => {
|
|
53475
|
+
const label = `bank ${bankId} observations_mission` + (entry.agents[0] !== bankId ? ` (${entry.agents.join(", ")})` : "");
|
|
53476
|
+
const firstAgent = entry.agents[0];
|
|
53477
|
+
if (!read.ok) {
|
|
53478
|
+
return {
|
|
53479
|
+
name: label,
|
|
53480
|
+
status: "warn",
|
|
53481
|
+
detail: `could not read observations_mission: ${read.reason}`
|
|
53482
|
+
};
|
|
53483
|
+
}
|
|
53484
|
+
const profileDefault = PROFILE_MEMORY_DEFAULTS[entry.config.extends ?? DEFAULT_PROFILE]?.observations_mission;
|
|
53485
|
+
const decision = decideObservationsMissionUpgrade(entry.config.memory?.observations_mission, profileDefault, read.mission);
|
|
53486
|
+
if (decision.action === "upgrade") {
|
|
53487
|
+
return {
|
|
53488
|
+
name: label,
|
|
53489
|
+
status: "warn",
|
|
53490
|
+
detail: read.mission == null || read.mission.trim() === "" ? "unset \u2014 this bank consolidates under Hindsight's stock mission, not switchroom's" : "carrying a superseded switchroom default",
|
|
53491
|
+
fix: `Run: switchroom agent reconcile ${firstAgent}`
|
|
53492
|
+
};
|
|
53493
|
+
}
|
|
53494
|
+
if (decision.action === "config") {
|
|
53495
|
+
return read.mission === decision.mission ? { name: label, status: "ok", detail: "set from switchroom.yaml" } : {
|
|
53496
|
+
name: label,
|
|
53497
|
+
status: "warn",
|
|
53498
|
+
detail: "switchroom.yaml sets a different observations_mission than the bank carries",
|
|
53499
|
+
fix: `Run: switchroom agent reconcile ${firstAgent}`
|
|
53500
|
+
};
|
|
53501
|
+
}
|
|
53502
|
+
const isSwitchroomDefault = read.mission === (profileDefault ?? DEFAULT_OBSERVATIONS_MISSION);
|
|
53503
|
+
return {
|
|
53504
|
+
name: label,
|
|
53505
|
+
status: "ok",
|
|
53506
|
+
detail: isSwitchroomDefault ? profileDefault != null ? `switchroom ${entry.config.extends ?? DEFAULT_PROFILE}-profile default` : "switchroom fleet default" : "operator-authored (left untouched by the never-clobber rule)"
|
|
53507
|
+
};
|
|
53508
|
+
});
|
|
53509
|
+
}
|
|
53044
53510
|
async function checkBankIngestHealth(config, url, opts) {
|
|
53045
53511
|
const results = [];
|
|
53046
53512
|
const now = opts?.now ?? new Date;
|
|
@@ -53200,6 +53666,7 @@ async function checkHindsight(config) {
|
|
|
53200
53666
|
results.push(checkHnswPartialIndexes());
|
|
53201
53667
|
results.push(await checkHindsightHealthEndpoint(url));
|
|
53202
53668
|
results.push(...await checkBankIngestHealth(config, url, { includeConsolidationBacklog: true }));
|
|
53669
|
+
results.push(...await checkBankObservationsMissions(config, url));
|
|
53203
53670
|
const memoryAgentsDir = resolveAgentsDir(config);
|
|
53204
53671
|
for (const agentName of Object.keys(config.agents)) {
|
|
53205
53672
|
results.push(checkAgentRecallHealth(agentName, resolve44(memoryAgentsDir, agentName)));
|
|
@@ -53314,7 +53781,7 @@ function checkPendingRetainsQueues(config, opts) {
|
|
|
53314
53781
|
}
|
|
53315
53782
|
}
|
|
53316
53783
|
const skippedNote = unreachable.length > 0 ? `; skipped (unreachable): ${unreachable.join(", ")}` : "";
|
|
53317
|
-
const backlogFix = `Clear a backlog explicitly (the SessionStart drain cannot \u2014 its per-entry ` + `timeout is clamped to the hook budget, far below a real retain): ` + `\`docker exec switchroom-<agent> python3 ` + `/state/agent/.claude/plugins/hindsight-memory/scripts/drain_pending.py --backlog\`. ` + `Run \`--phase reconcile\` first \u2014 it costs nothing and typically clears most of ` + `the queue by confirming those documents already exist. PACE THE REST: the retain ` + `model pool is small and shared with live traffic, so drain ONE agent at a time at ` + `the default HINDSIGHT_DRAIN_CONCURRENCY=1, and keep HINDSIGHT_DRAIN_SLEEP_S set. ` + `Point HINDSIGHT_DRAIN_P95_CMD at the same figure the latency watchdog alarms on so ` + `the replay backs off instead of causing the alarm \u2014 on a LiteLLM deployment that is ` + `\`docker exec -i <postgres> psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -c ` + `"select coalesce((select round(percentile_cont(0.95) within group (order by ` + `request_duration_ms)) from \\"LiteLLM_SpendLogs\\" where \\"startTime\\" > now() ` + `- interval '10 minutes' and metadata->>'user_api_key_alias' like 'hindsight-%' and ` + `status='success' and request_duration_ms is not null), -1)"\` (must print a ` + `millisecond integer on stdout; unset = no backoff). Fix the upstream first (bank ` + `health rows above), or every retry just ages entries toward .dead. Entries the ` + `drain retires are MOVED to \`.hindsight/pending-reconciled/\` (bounded, so they ` + `expire), never deleted \u2014 every retire rests on an HTTP 200, which is an ack and ` + `not proof, so the payload stays recoverable. If that archive cannot be written ` + `(disk full, permissions) the entry STAYS QUEUED and the drain says so on stderr; ` + `a failure to archive is never a reason to delete. FIX THE DISK ANYWAY \u2014 "stays ` + `queued" is bounded: entries pile up, the queue hits ` + `HINDSIGHT_PENDING_MAX_ENTRIES/MAX_BYTES, and enqueue then sheds the OLDEST ` + `entries to keep accepting the newest. While the disk is full their archive move ` + `fails too, so those oldest turns are removed outright (the ledger line reads ` + `\`reason=...+archive-failed\`). Under sustained ENOSPC "keep it queued" degrades ` + `to "keep the newest, drop the oldest".`;
|
|
53784
|
+
const backlogFix = `Clear a backlog explicitly (the SessionStart drain cannot \u2014 its per-entry ` + `timeout is clamped to the hook budget, far below a real retain): ` + `\`docker exec switchroom-<agent> python3 ` + `/state/agent/.claude/plugins/hindsight-memory/scripts/drain_pending.py --backlog\`. ` + `Run \`--phase reconcile\` first \u2014 it costs nothing and typically clears most of ` + `the queue by confirming those documents already exist. PACE THE REST: the retain ` + `model pool is small and shared with live traffic, so drain ONE agent at a time at ` + `the default HINDSIGHT_DRAIN_CONCURRENCY=1, and keep HINDSIGHT_DRAIN_SLEEP_S set. ` + `Point HINDSIGHT_DRAIN_P95_CMD at the same figure the latency watchdog alarms on so ` + `the replay backs off instead of causing the alarm \u2014 on a LiteLLM deployment that is ` + `\`docker exec -i <postgres> psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -c ` + `"select coalesce((select round(percentile_cont(0.95) within group (order by ` + `request_duration_ms)) from \\"LiteLLM_SpendLogs\\" where \\"startTime\\" > now() ` + `- interval '10 minutes' and metadata->>'user_api_key_alias' like 'hindsight-%' and ` + `status='success' and request_duration_ms is not null), -1)"\` (must print a ` + `millisecond integer on stdout; unset = no backoff). ` + `SAFE ALONGSIDE THE \`hindsight-drain\` SIDECAR and a session booting underneath ` + `you: drain_pending.py takes an exclusive lock on ` + `\`$HOME/.hindsight/drain-pending.lock\` for the whole run, so whichever drain ` + `starts second prints "another drain holds ..." and exits without touching the ` + `queue. Do NOT wrap this in \`flock\` on that path yourself \u2014 you would hold the ` + `lock the script then asks for, and it would skip every time. Entries past the ` + `attempt ceiling (HINDSIGHT_DRAIN_ATTEMPT_CEILING, default 20) are PARKED and ` + `reported rather than retried forever; add \`--force\` once the upstream is fixed ` + `to replay them. ` + `Fix the upstream first (bank ` + `health rows above), or every retry just ages entries toward .dead. Entries the ` + `drain retires are MOVED to \`.hindsight/pending-reconciled/\` (bounded, so they ` + `expire), never deleted \u2014 every retire rests on an HTTP 200, which is an ack and ` + `not proof, so the payload stays recoverable. If that archive cannot be written ` + `(disk full, permissions) the entry STAYS QUEUED and the drain says so on stderr; ` + `a failure to archive is never a reason to delete. FIX THE DISK ANYWAY \u2014 "stays ` + `queued" is bounded: entries pile up, the queue hits ` + `HINDSIGHT_PENDING_MAX_ENTRIES/MAX_BYTES, and enqueue then sheds the OLDEST ` + `entries to keep accepting the newest. While the disk is full their archive move ` + `fails too, so those oldest turns are removed outright (the ledger line reads ` + `\`reason=...+archive-failed\`). Under sustained ENOSPC "keep it queued" degrades ` + `to "keep the newest, drop the oldest".`;
|
|
53318
53785
|
if (dead.length > 0 || dropped.length > 0 || evicted.length > 0) {
|
|
53319
53786
|
const parts = [];
|
|
53320
53787
|
if (dead.length > 0)
|
|
@@ -54506,6 +54973,25 @@ function registerDoctorCommand(program3) {
|
|
|
54506
54973
|
},
|
|
54507
54974
|
{ title: "MFF Skill", results: await checkMff(passphrase, vaultPath, config) },
|
|
54508
54975
|
{ title: "Webkite", results: runWebkiteChecks(config) },
|
|
54976
|
+
{
|
|
54977
|
+
title: "OpenRouter Credit",
|
|
54978
|
+
results: await runOpenRouterCreditChecks({
|
|
54979
|
+
enabled: usesOpenRouter(readLitellmConfigText()),
|
|
54980
|
+
readSecret: async (key) => {
|
|
54981
|
+
try {
|
|
54982
|
+
const { getViaBrokerStructured: getViaBrokerStructured2 } = await Promise.resolve().then(() => (init_client(), exports_client));
|
|
54983
|
+
const result = await getViaBrokerStructured2(key);
|
|
54984
|
+
if (result.kind === "ok" && result.entry.kind === "string") {
|
|
54985
|
+
return result.entry.value;
|
|
54986
|
+
}
|
|
54987
|
+
return null;
|
|
54988
|
+
} catch {
|
|
54989
|
+
return null;
|
|
54990
|
+
}
|
|
54991
|
+
},
|
|
54992
|
+
fetchFn: (url, init) => fetch(url, { headers: init?.headers, signal: init?.signal ?? AbortSignal.timeout(1e4) })
|
|
54993
|
+
})
|
|
54994
|
+
},
|
|
54509
54995
|
{ title: "Cron Session", results: runCronSessionChecks(config) },
|
|
54510
54996
|
{
|
|
54511
54997
|
title: "Generated-surface drift (KEN-130)",
|
|
@@ -54573,6 +55059,7 @@ var init_doctor = __esm(() => {
|
|
|
54573
55059
|
init_scaffold();
|
|
54574
55060
|
init_manager();
|
|
54575
55061
|
init_accounts();
|
|
55062
|
+
init_schema();
|
|
54576
55063
|
init_manifest();
|
|
54577
55064
|
init_hindsight2();
|
|
54578
55065
|
init_hindsight();
|
|
@@ -54589,6 +55076,7 @@ var init_doctor = __esm(() => {
|
|
|
54589
55076
|
init_doctor_hostd();
|
|
54590
55077
|
init_doctor_drive();
|
|
54591
55078
|
init_doctor_webkite();
|
|
55079
|
+
init_doctor_openrouter_credit();
|
|
54592
55080
|
init_doctor_cron_session();
|
|
54593
55081
|
init_doctor_flood_pressure();
|
|
54594
55082
|
init_doctor_drift();
|
|
@@ -101885,25 +102373,25 @@ function syncIssue(deps, repo, job_spec, iss) {
|
|
|
101885
102373
|
}
|
|
101886
102374
|
return;
|
|
101887
102375
|
}
|
|
101888
|
-
const
|
|
101889
|
-
const upd = deps.run(["issue", "edit",
|
|
102376
|
+
const num2 = String(iss.gh_issue);
|
|
102377
|
+
const upd = deps.run(["issue", "edit", num2, "-R", repo, "--body", body]);
|
|
101890
102378
|
if (!upd.ok) {
|
|
101891
|
-
deps.log(`fleet-health: gh issue edit #${
|
|
102379
|
+
deps.log(`fleet-health: gh issue edit #${num2} failed: ${upd.stderr}`);
|
|
101892
102380
|
}
|
|
101893
102381
|
if (iss.status === "closed") {
|
|
101894
102382
|
const c = deps.run([
|
|
101895
102383
|
"issue",
|
|
101896
102384
|
"close",
|
|
101897
|
-
|
|
102385
|
+
num2,
|
|
101898
102386
|
"-R",
|
|
101899
102387
|
repo,
|
|
101900
102388
|
"--comment",
|
|
101901
102389
|
`Verified count-drop (frequency ${iss.frequency} \u2264 resolved threshold). Closed by the Fleet Health sensor.`
|
|
101902
102390
|
]);
|
|
101903
102391
|
if (c.ok)
|
|
101904
|
-
deps.log(`fleet-health: closed GH #${
|
|
102392
|
+
deps.log(`fleet-health: closed GH #${num2} (count-drop) for ${iss.dedup_key}`);
|
|
101905
102393
|
else
|
|
101906
|
-
deps.log(`fleet-health: gh issue close #${
|
|
102394
|
+
deps.log(`fleet-health: gh issue close #${num2} failed: ${c.stderr}`);
|
|
101907
102395
|
}
|
|
101908
102396
|
}
|
|
101909
102397
|
function syncLedgerIssues(ledger, repo, deps) {
|
|
@@ -103275,6 +103763,278 @@ async function notifyOperator(agentsDir, text, log) {
|
|
|
103275
103763
|
return agent !== null;
|
|
103276
103764
|
}
|
|
103277
103765
|
|
|
103766
|
+
// src/cli/openrouter-watch.ts
|
|
103767
|
+
init_source();
|
|
103768
|
+
import { existsSync as existsSync113, readdirSync as readdirSync46 } from "node:fs";
|
|
103769
|
+
import { homedir as homedir64 } from "node:os";
|
|
103770
|
+
import { join as join112, resolve as resolve65 } from "node:path";
|
|
103771
|
+
|
|
103772
|
+
// src/openrouter/install-cron.ts
|
|
103773
|
+
import { existsSync as existsSync112, mkdirSync as mkdirSync63, readFileSync as readFileSync99, renameSync as renameSync28, writeFileSync as writeFileSync47 } from "node:fs";
|
|
103774
|
+
import { dirname as dirname42 } from "node:path";
|
|
103775
|
+
var CRON_PATH2 = "/etc/cron.d/openrouter-watch";
|
|
103776
|
+
var CRON_SCHEDULE2 = "17 * * * *";
|
|
103777
|
+
var CRON_LOG_PATH2 = "/var/log/openrouter-watch.log";
|
|
103778
|
+
var CRON_LOCK_PATH2 = "/run/lock/openrouter-watch.lock";
|
|
103779
|
+
function renderCron2(opts) {
|
|
103780
|
+
return `# switchroom openrouter-watch \u2014 model-free OpenRouter credit watchdog.
|
|
103781
|
+
` + `# Managed by \`switchroom openrouter-watch --install-cron\`; edits are overwritten.
|
|
103782
|
+
` + `# Exit 0 = clean, 10 = credit signal firing (DM sent), 1 = the check could not complete.
|
|
103783
|
+
` + `SHELL=/bin/sh
|
|
103784
|
+
` + `PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
|
103785
|
+
` + `${CRON_SCHEDULE2} ${opts.user} /usr/bin/flock -n ${CRON_LOCK_PATH2} ` + `${opts.binary} openrouter-watch >> ${CRON_LOG_PATH2} 2>&1
|
|
103786
|
+
`;
|
|
103787
|
+
}
|
|
103788
|
+
function installCron2(opts) {
|
|
103789
|
+
const path10 = opts.path ?? CRON_PATH2;
|
|
103790
|
+
const content = renderCron2(opts);
|
|
103791
|
+
if (existsSync112(path10)) {
|
|
103792
|
+
try {
|
|
103793
|
+
if (readFileSync99(path10, "utf8") === content) {
|
|
103794
|
+
return { status: "unchanged", path: path10, content };
|
|
103795
|
+
}
|
|
103796
|
+
} catch {}
|
|
103797
|
+
}
|
|
103798
|
+
mkdirSync63(dirname42(path10), { recursive: true });
|
|
103799
|
+
const tmp = `${path10}.${process.pid}.tmp`;
|
|
103800
|
+
writeFileSync47(tmp, content, { mode: 420 });
|
|
103801
|
+
renameSync28(tmp, path10);
|
|
103802
|
+
return { status: "installed", path: path10, content };
|
|
103803
|
+
}
|
|
103804
|
+
// src/openrouter/state.ts
|
|
103805
|
+
import { mkdirSync as mkdirSync64, readFileSync as readFileSync100, renameSync as renameSync29, writeFileSync as writeFileSync48 } from "node:fs";
|
|
103806
|
+
import { homedir as homedir63 } from "node:os";
|
|
103807
|
+
import { dirname as dirname43, resolve as resolve64 } from "node:path";
|
|
103808
|
+
function defaultStatePath3(home2 = process.env.SWITCHROOM_HOME ?? process.env.HOME ?? homedir63()) {
|
|
103809
|
+
return resolve64(home2, ".switchroom", "openrouter-watch", "state.json");
|
|
103810
|
+
}
|
|
103811
|
+
function loadState2(path10) {
|
|
103812
|
+
let parsed;
|
|
103813
|
+
try {
|
|
103814
|
+
parsed = JSON.parse(readFileSync100(path10, "utf8"));
|
|
103815
|
+
} catch {
|
|
103816
|
+
return {};
|
|
103817
|
+
}
|
|
103818
|
+
if (typeof parsed !== "object" || parsed === null)
|
|
103819
|
+
return {};
|
|
103820
|
+
const s = parsed;
|
|
103821
|
+
if (s.v !== 1)
|
|
103822
|
+
return {};
|
|
103823
|
+
const out = {};
|
|
103824
|
+
if (typeof s.lastNotifiedAt === "number" && Number.isFinite(s.lastNotifiedAt)) {
|
|
103825
|
+
out.lastNotifiedAt = s.lastNotifiedAt;
|
|
103826
|
+
}
|
|
103827
|
+
if (typeof s.lastNotifiedStatus === "string")
|
|
103828
|
+
out.lastNotifiedStatus = s.lastNotifiedStatus;
|
|
103829
|
+
if (out.lastNotifiedAt === undefined)
|
|
103830
|
+
delete out.lastNotifiedStatus;
|
|
103831
|
+
return out;
|
|
103832
|
+
}
|
|
103833
|
+
function saveState2(path10, state) {
|
|
103834
|
+
mkdirSync64(dirname43(path10), { recursive: true, mode: 448 });
|
|
103835
|
+
const tmp = `${path10}.${process.pid}.tmp`;
|
|
103836
|
+
const payload = { v: 1, ...state };
|
|
103837
|
+
writeFileSync48(tmp, JSON.stringify(payload, null, 2) + `
|
|
103838
|
+
`, { mode: 384 });
|
|
103839
|
+
renameSync29(tmp, path10);
|
|
103840
|
+
}
|
|
103841
|
+
|
|
103842
|
+
// src/openrouter/watch.ts
|
|
103843
|
+
var RENOTIFY_MS2 = 6 * 60 * 60 * 1000;
|
|
103844
|
+
function decideNotification(assessment, state, now) {
|
|
103845
|
+
if (!assessment.actionable) {
|
|
103846
|
+
return { notice: null, nextState: {}, firing: false };
|
|
103847
|
+
}
|
|
103848
|
+
const escalated = state.lastNotifiedStatus != null && state.lastNotifiedStatus !== "fail" && assessment.status === "fail";
|
|
103849
|
+
const cooled = state.lastNotifiedAt == null || now - state.lastNotifiedAt >= RENOTIFY_MS2;
|
|
103850
|
+
if (!escalated && !cooled) {
|
|
103851
|
+
return { notice: null, nextState: state, firing: true };
|
|
103852
|
+
}
|
|
103853
|
+
const repeat = state.lastNotifiedAt != null && !escalated;
|
|
103854
|
+
return {
|
|
103855
|
+
notice: renderNotice(assessment, { repeat, escalated }),
|
|
103856
|
+
nextState: { lastNotifiedAt: now, lastNotifiedStatus: assessment.status },
|
|
103857
|
+
firing: true
|
|
103858
|
+
};
|
|
103859
|
+
}
|
|
103860
|
+
function renderNotice(assessment, opts) {
|
|
103861
|
+
const head = assessment.status === "fail" ? opts.escalated ? "\uD83D\uDEA8 OpenRouter credit \u2014 ESCALATED" : "\uD83D\uDEA8 OpenRouter credit" : "\u26a0\ufe0f OpenRouter credit";
|
|
103862
|
+
const suffix = opts.repeat && !opts.escalated ? " (still unresolved)" : "";
|
|
103863
|
+
return [
|
|
103864
|
+
`${head}${suffix}`,
|
|
103865
|
+
assessment.detail,
|
|
103866
|
+
assessment.fix ? `\u2192 ${assessment.fix}` : ""
|
|
103867
|
+
].filter((l) => l.length > 0).join(`
|
|
103868
|
+
`);
|
|
103869
|
+
}
|
|
103870
|
+
|
|
103871
|
+
// src/openrouter/run.ts
|
|
103872
|
+
async function tick2(deps) {
|
|
103873
|
+
const now = deps.now?.() ?? Date.now();
|
|
103874
|
+
let apiKey = null;
|
|
103875
|
+
try {
|
|
103876
|
+
apiKey = await deps.readSecret(OPENROUTER_VAULT_KEY);
|
|
103877
|
+
} catch (err2) {
|
|
103878
|
+
deps.log(`openrouter-watch: vault read failed for \`${OPENROUTER_VAULT_KEY}\`: ${err2.message}`);
|
|
103879
|
+
}
|
|
103880
|
+
if (!apiKey) {
|
|
103881
|
+
deps.log(`openrouter-watch: vault key \`${OPENROUTER_VAULT_KEY}\` is not readable \u2014 credit cannot be checked.`);
|
|
103882
|
+
return {
|
|
103883
|
+
assessment: {
|
|
103884
|
+
status: "skip",
|
|
103885
|
+
detail: `Vault key \`${OPENROUTER_VAULT_KEY}\` unreadable.`,
|
|
103886
|
+
actionable: false
|
|
103887
|
+
},
|
|
103888
|
+
notice: null,
|
|
103889
|
+
firing: false,
|
|
103890
|
+
delivered: false,
|
|
103891
|
+
exitCode: 1
|
|
103892
|
+
};
|
|
103893
|
+
}
|
|
103894
|
+
const fetched = await fetchKeySnapshot(apiKey, deps.fetchFn);
|
|
103895
|
+
const assessment = fetched.kind === "ok" ? evaluateCredit(fetched.snapshot) : assessFetchFailure(fetched);
|
|
103896
|
+
const prior = loadState2(deps.statePath);
|
|
103897
|
+
const decision = decideNotification(assessment, prior, now);
|
|
103898
|
+
let delivered = false;
|
|
103899
|
+
if (decision.notice && !deps.dryRun) {
|
|
103900
|
+
delivered = await deps.notify(decision.notice);
|
|
103901
|
+
}
|
|
103902
|
+
if (!deps.dryRun && (decision.notice === null || delivered)) {
|
|
103903
|
+
try {
|
|
103904
|
+
saveState2(deps.statePath, decision.nextState);
|
|
103905
|
+
} catch (err2) {
|
|
103906
|
+
deps.log(`openrouter-watch: could not persist state: ${err2.message}`);
|
|
103907
|
+
return { assessment, notice: decision.notice, firing: decision.firing, delivered, exitCode: 1 };
|
|
103908
|
+
}
|
|
103909
|
+
}
|
|
103910
|
+
if (decision.notice && !deps.dryRun && !delivered) {
|
|
103911
|
+
deps.log("openrouter-watch: alert reached no gateway \u2014 undelivered, will retry next tick");
|
|
103912
|
+
return { assessment, notice: decision.notice, firing: decision.firing, delivered, exitCode: 1 };
|
|
103913
|
+
}
|
|
103914
|
+
const cannotComplete = fetched.kind === "unreachable";
|
|
103915
|
+
const exitCode = cannotComplete ? 1 : decision.firing ? 10 : 0;
|
|
103916
|
+
return { assessment, notice: decision.notice, firing: decision.firing, delivered, exitCode };
|
|
103917
|
+
}
|
|
103918
|
+
// src/cli/openrouter-watch.ts
|
|
103919
|
+
function registerOpenRouterWatchCommand(program3) {
|
|
103920
|
+
program3.command("openrouter-watch").description("Model-free OpenRouter credit watchdog: warns before the key's credit runs out. " + "Alerts at most once per 6h, and escalates immediately.").option("--install-cron", `arm the watchdog: write ${CRON_PATH2} (${CRON_SCHEDULE2}, flock-guarded) and exit`, false).option("--cron-user <user>", "unix user the cron tick runs as (default: current user)").option("--dry-run", "evaluate and print; send no DM and write no state", false).option("--json", "emit machine-readable JSON", false).option("--state <path>", "state file path (default ~/.switchroom/openrouter-watch/state.json)").option("--agents-dir <path>", "agents scaffold dir (default ~/.switchroom/agents)").action(async (opts) => {
|
|
103921
|
+
if (opts.installCron) {
|
|
103922
|
+
process.exitCode = runInstallCron2(opts.cronUser);
|
|
103923
|
+
return;
|
|
103924
|
+
}
|
|
103925
|
+
const agentsDir = opts.agentsDir ?? process.env.SWITCHROOM_AGENTS_DIR ?? join112(process.env.SWITCHROOM_HOME ?? process.env.HOME ?? homedir64(), ".switchroom", "agents");
|
|
103926
|
+
const statePath = opts.state ?? defaultStatePath3();
|
|
103927
|
+
const log = (m) => {
|
|
103928
|
+
process.stderr.write(`${m}
|
|
103929
|
+
`);
|
|
103930
|
+
};
|
|
103931
|
+
const result = await tick2({
|
|
103932
|
+
statePath,
|
|
103933
|
+
dryRun: Boolean(opts.dryRun),
|
|
103934
|
+
log,
|
|
103935
|
+
readSecret: async (key) => {
|
|
103936
|
+
const { getViaBrokerStructured: getViaBrokerStructured2 } = await Promise.resolve().then(() => (init_client(), exports_client));
|
|
103937
|
+
const res = await getViaBrokerStructured2(key);
|
|
103938
|
+
if (res.kind === "ok" && res.entry.kind === "string")
|
|
103939
|
+
return res.entry.value;
|
|
103940
|
+
return null;
|
|
103941
|
+
},
|
|
103942
|
+
fetchFn: (url, init) => fetch(url, {
|
|
103943
|
+
headers: init?.headers,
|
|
103944
|
+
signal: init?.signal ?? AbortSignal.timeout(1e4)
|
|
103945
|
+
}),
|
|
103946
|
+
notify: (text) => notifyOperator2(agentsDir, text, log)
|
|
103947
|
+
});
|
|
103948
|
+
if (opts.json) {
|
|
103949
|
+
process.stdout.write(JSON.stringify({
|
|
103950
|
+
status: result.assessment.status,
|
|
103951
|
+
detail: result.assessment.detail,
|
|
103952
|
+
fix: result.assessment.fix ?? null,
|
|
103953
|
+
actionable: result.assessment.actionable,
|
|
103954
|
+
firing: result.firing,
|
|
103955
|
+
notified: result.notice,
|
|
103956
|
+
delivered: result.delivered,
|
|
103957
|
+
exitCode: result.exitCode,
|
|
103958
|
+
vaultKey: OPENROUTER_VAULT_KEY,
|
|
103959
|
+
thresholds: {
|
|
103960
|
+
warnRemainingFraction: WARN_REMAINING_FRACTION,
|
|
103961
|
+
warnRemainingUsd: WARN_REMAINING_USD,
|
|
103962
|
+
failRemainingFraction: FAIL_REMAINING_FRACTION,
|
|
103963
|
+
failRemainingUsd: FAIL_REMAINING_USD,
|
|
103964
|
+
renotifyMs: RENOTIFY_MS2
|
|
103965
|
+
}
|
|
103966
|
+
}, null, 2) + `
|
|
103967
|
+
`);
|
|
103968
|
+
} else {
|
|
103969
|
+
const mark = result.assessment.status === "fail" ? source_default.red("FAIL") : result.assessment.status === "warn" ? source_default.yellow("WARN") : result.assessment.status === "skip" ? source_default.gray("skip") : source_default.green("ok ");
|
|
103970
|
+
process.stdout.write(`${mark} ${result.assessment.detail}
|
|
103971
|
+
`);
|
|
103972
|
+
if (result.assessment.fix)
|
|
103973
|
+
process.stdout.write(` \u2192 ${result.assessment.fix}
|
|
103974
|
+
`);
|
|
103975
|
+
if (result.notice) {
|
|
103976
|
+
const header = opts.dryRun ? "would send" : result.delivered ? "sent" : "UNDELIVERED";
|
|
103977
|
+
process.stdout.write(`
|
|
103978
|
+
${source_default.bold(`operator DM ${header}:`)}
|
|
103979
|
+
`);
|
|
103980
|
+
process.stdout.write(result.notice.split(`
|
|
103981
|
+
`).map((l) => ` | ${l}`).join(`
|
|
103982
|
+
`) + `
|
|
103983
|
+
`);
|
|
103984
|
+
} else if (result.firing) {
|
|
103985
|
+
process.stdout.write(source_default.gray(` (firing, but already reported within the ${RENOTIFY_MS2 / 3600000}h window)
|
|
103986
|
+
`));
|
|
103987
|
+
}
|
|
103988
|
+
}
|
|
103989
|
+
process.exitCode = result.exitCode;
|
|
103990
|
+
});
|
|
103991
|
+
}
|
|
103992
|
+
function runInstallCron2(cronUser) {
|
|
103993
|
+
const user = cronUser ?? process.env.SUDO_USER ?? process.env.USER ?? process.env.LOGNAME;
|
|
103994
|
+
if (!user || user === "root") {
|
|
103995
|
+
process.stderr.write(source_default.red("openrouter-watch: refusing to install a cron for `root`.\n") + " The state file and the agents scaffold live under the OPERATOR's " + "~/.switchroom, so a root tick would keep its re-notify ledger somewhere " + `nothing else reads.
|
|
103996
|
+
` + " Re-run with `--cron-user <operator>`.\n");
|
|
103997
|
+
return 1;
|
|
103998
|
+
}
|
|
103999
|
+
const binary = process.env.SWITCHROOM_BINARY ?? "/usr/local/bin/switchroom";
|
|
104000
|
+
if (!binary.startsWith("/")) {
|
|
104001
|
+
process.stderr.write(source_default.red(`openrouter-watch: SWITCHROOM_BINARY must be an absolute path (got ${binary})
|
|
104002
|
+
`));
|
|
104003
|
+
return 1;
|
|
104004
|
+
}
|
|
104005
|
+
let res;
|
|
104006
|
+
try {
|
|
104007
|
+
res = installCron2({ user, binary });
|
|
104008
|
+
} catch (e) {
|
|
104009
|
+
process.stderr.write(source_default.red(`openrouter-watch: could not write ${CRON_PATH2}: ${e.message}
|
|
104010
|
+
`) + " This path needs root; re-run under sudo with `--cron-user <operator>`.\n");
|
|
104011
|
+
return 1;
|
|
104012
|
+
}
|
|
104013
|
+
const verb = res.status === "installed" ? "installed" : "already up to date";
|
|
104014
|
+
process.stdout.write(`${source_default.green("\u2713")} openrouter-watch cron ${verb} at ${res.path} ` + `(${CRON_SCHEDULE2}, as ${user})
|
|
104015
|
+
` + ` Verify with: switchroom openrouter-watch --dry-run
|
|
104016
|
+
`);
|
|
104017
|
+
return 0;
|
|
104018
|
+
}
|
|
104019
|
+
async function notifyOperator2(agentsDir, text, log) {
|
|
104020
|
+
const candidates = [];
|
|
104021
|
+
try {
|
|
104022
|
+
for (const name of readdirSync46(agentsDir).sort()) {
|
|
104023
|
+
const sock = resolve65(agentsDir, name, "telegram", "gateway.sock");
|
|
104024
|
+
if (existsSync113(sock))
|
|
104025
|
+
candidates.push({ agent: name, sock });
|
|
104026
|
+
}
|
|
104027
|
+
} catch (e) {
|
|
104028
|
+
log(`openrouter-watch: agents dir unreadable (${e.message})`);
|
|
104029
|
+
}
|
|
104030
|
+
if (candidates.length === 0) {
|
|
104031
|
+
log("openrouter-watch: no gateway socket found \u2014 alert undelivered, will retry next tick");
|
|
104032
|
+
return false;
|
|
104033
|
+
}
|
|
104034
|
+
const agent = await postOperatorNoticeViaGateways(candidates, text, log, 5000, "openrouter-watch", "");
|
|
104035
|
+
return agent !== null;
|
|
104036
|
+
}
|
|
104037
|
+
|
|
103278
104038
|
// src/cli/index.ts
|
|
103279
104039
|
init_posthog();
|
|
103280
104040
|
installGlobalErrorHandlers();
|
|
@@ -103334,6 +104094,7 @@ registerWebdCommand(program3);
|
|
|
103334
104094
|
registerHostCommand(program3);
|
|
103335
104095
|
registerFleetHealthCommand(program3);
|
|
103336
104096
|
registerHindsightWatchCommand(program3);
|
|
104097
|
+
registerOpenRouterWatchCommand(program3);
|
|
103337
104098
|
|
|
103338
104099
|
// bin/switchroom.ts
|
|
103339
104100
|
program3.parse();
|