pi-lemonade-link 1.0.2 → 1.1.0

package/README.md

@@ -375,6 +375,7 @@ The full shape (values shown are the built-in defaults):
   "classifyPath": "/v1/classify",
   "beaconPort": 13305,
   "chatOnly": true,
+  "cacheOnOutage": false,
   "defaultTranscriptionModel": "Whisper-Large-v3",
   "defaultImageModel": "",
   "defaultUpscaleModel": "RealESRGAN-x4plus",
@@ -403,6 +404,7 @@ The full shape (values shown are the built-in defaults):
 | `apiKey` | Shared fallback `Authorization: Bearer …` for instances without their own. A dummy value works for auth-less servers (pi requires non-empty auth to list models). |
 | `*Path` | API endpoint paths, in case a future lemonade changes them. |
 | `chatOnly` | Filter the provider down to chat-capable models. |
+| `cacheOnOutage` | **Outage cache**: when a probe can't reach an instance, `/model` lists its last good catalog — persisted on every successful probe, provenance-marked `(cached <time>)`, warned at session start, and re-synced automatically (prompt submits, model selections, session starts, plus the status bar's own health poll) with a `+new −gone` diff notice once the box answers. Once an instance is known down (durable flag from the last failed probe), the next startup serves the cached listing **instantly — no probe wait** and reconciles in the background, so the down-box stall only ever bites once per outage, at the transition into it. `false` (default) keeps the fully-honest empty listing. Manage via `/lemonade-setup` → *Catalog cache*; force a re-probe with *Refresh connection*. |
 | `defaultTranscriptionModel` / `defaultImageModel` / `defaultUpscaleModel` / `defaultClassifierModel` | Fixed model ids for the tools. Empty = auto-pick from the live catalog (by label, or by `onnxruntime` recipe for classifiers). |
 | `outputDir` | Where generated files (images, audio, meshes) are saved. Empty = agent's working directory. |
 | `*TimeoutMs` | Per-operation timeouts. `pullTimeoutMs` only caps the *blocking* pull fallback — normal pulls use download jobs with live progress. |
@@ -432,6 +434,18 @@ WAN, VPN down, box asleep):
   (`Error: Model "lemonade/<id>" not found`), and resume working once the box
   is back and a Refresh has re-registered it. Non-reachable chat requests are
   the honest outcome; phantom listings are never shown.
+- **`cacheOnOutage: true` softens the offline cliff**: instead of the empty
+  listing, `/model` shows the instance's last good catalog — provenance-marked
+  `(cached <time>)` in the model name, warned once per session start, and
+  automatically re-synced (with a `+new −gone` diff notice) on prompt submits,
+  model selections, session starts, and the status bar's own health poll.
+  Pinned sessions then resolve and soft-fail at send time instead of
+  hard-failing model resolution. `/lemonade-setup` → *Refresh connection*
+  forces an immediate re-probe; *Catalog cache* shows/clears the snapshots.
+  Once an instance is known down, the flag makes the next startup **instant**:
+  the cached listing is served with zero probe wait and reconciled by a
+  background probe — the `discoveryTimeoutMs` stall only happens at the
+  transition into an outage, never on startups after it.
 - **Tools fail with actionable text, not cryptic errors**: every tool detects
   the unreachable server and returns the same message — *where* it tried to
   connect and *what to do about it* (reconnect, or point the extension at a

package/index.ts

@@ -108,6 +108,11 @@ interface LemonadeConfig {
   // info notice, deduplicated to once per (instance, new release) — the
   // TUI always shows the live truth. Separately toggleable; default on.
   upstreamVersionCheck: boolean;
+  // Outage cache: when a probe can't reach an instance, register its last
+  // good catalog (persisted on every successful probe) instead of nothing —
+  // provenance-marked "(cached <time>)" in /model, warned at session start,
+  // and re-synced with a diff notice once the box answers again.
+  cacheOnOutage: boolean;
   discoveryTimeoutMs: number;
   beaconTimeoutMs: number;
   loadTimeoutMs: number;
@@ -156,6 +161,7 @@ const DEFAULT_CONFIG: LemonadeConfig = {
   classifyPath: "/v1/classify",
   beaconPort: 13305,
   chatOnly: true,
+  cacheOnOutage: false,
   defaultTranscriptionModel: "Whisper-Large-v3",
   defaultImageModel: "",
   defaultUpscaleModel: "RealESRGAN-x4plus",
@@ -752,6 +758,105 @@ function saveUpstreamSnapshot(snap: UpstreamVersionSnapshot): void {
   }
 }
 
+// ─── Outage catalog cache (cacheOnOutage) ────────────────────────────────
+
+interface CatalogCacheEntry {
+  fetchedAt: number; // epoch ms of the successful probe this snapshot came from
+  catalog: LemonadeModel[];
+  health: LemonadeHealth | null;
+  /** Durable last-known-state: the most recent probe pass for this instance
+   *  FAILED. Gates instant cache-serving (hybrid SWR) at the next activation —
+   *  an instance known down serves its cached listing with zero probe wait,
+ *  then reconciles in the background. Cleared on every successful probe.
+   *  Only meaningful alongside a catalog: a snapshotless instance is never
+   *  flagged (there is nothing to serve anyway). */
+  unreachableAtLastProbe?: boolean;
+}
+
+interface CatalogCacheFile {
+  instances: Record<string, CatalogCacheEntry>;
+}
+
+function catalogCachePath(): string {
+  return path.join(getAgentDir(), "cache", "pi-lemonade-link", "catalog-cache.json");
+}
+
+/** Defensively validated load — a corrupt/absent cache is an empty cache,
+ *  never a load failure. */
+function loadCatalogCache(): CatalogCacheFile {
+  try {
+    const raw = JSON.parse(fs.readFileSync(catalogCachePath(), "utf8")) as CatalogCacheFile;
+    if (!raw || typeof raw !== "object" || typeof raw.instances !== "object" || raw.instances === null) {
+      return { instances: {} };
+    }
+    const instances: Record<string, CatalogCacheEntry> = {};
+    for (const [name, e] of Object.entries(raw.instances)) {
+      if (
+        e && typeof e === "object" && typeof e.fetchedAt === "number" && Array.isArray(e.catalog) &&
+        e.catalog.every((m) => m && typeof (m as LemonadeModel).id === "string")
+      ) {
+        instances[name] = {
+          fetchedAt: e.fetchedAt,
+          catalog: e.catalog as LemonadeModel[],
+          health: (e.health ?? null) as LemonadeHealth | null,
+          ...(e.unreachableAtLastProbe === true ? { unreachableAtLastProbe: true } : {}),
+        };
+      }
+    }
+    return { instances };
+  } catch {
+    return { instances: {} };
+  }
+}
+
+function saveCatalogCacheEntry(name: string, entry: CatalogCacheEntry): void {
+  try {
+    const cache = loadCatalogCache();
+    cache.instances[name] = entry;
+    fs.mkdirSync(path.dirname(catalogCachePath()), { recursive: true });
+    fs.writeFileSync(catalogCachePath(), JSON.stringify(cache, null, 2) + "\n");
+  } catch {
+    /* cache dir unwritable → the outage fallback simply has nothing to serve */
+  }
+}
+
+/** Mark the durable last-known-state WITHOUT touching the snapshot: the
+ *  most recent probe for this instance failed (true) / succeeded (false).
+ *  Keeps the catalog/fetchedAt/health of the last successful probe — the
+ *  flag and the snapshot age independently by design. */
+function flagCatalogCacheEntry(name: string, unreachableAtLastProbe: boolean): void {
+  try {
+    const cache = loadCatalogCache();
+    const entry = cache.instances[name];
+    if (!entry) return; // no snapshot → nothing to serve, flag would be meaningless
+    entry.unreachableAtLastProbe = unreachableAtLastProbe;
+    fs.mkdirSync(path.dirname(catalogCachePath()), { recursive: true });
+    fs.writeFileSync(catalogCachePath(), JSON.stringify(cache, null, 2) + "\n");
+  } catch {
+    /* unwritable cache — the next activation just probes live instead */
+  }
+}
+
+function clearCatalogCache(): void {
+  try {
+    fs.rmSync(catalogCachePath(), { force: true });
+  } catch {
+    /* nothing to clear / unwritable — loadCatalogCache still returns {} */
+  }
+}
+
+/** Human stamp for a cached snapshot: "14:32" same-day, "Sep 12 14:32"
+ *  older. The age IS the provenance — users discount old snapshots themselves. */
+function formatCacheStamp(ts: number): string {
+  const d = new Date(ts);
+  const pad = (n: number) => String(n).padStart(2, "0");
+  const hms = `${pad(d.getHours())}:${pad(d.getMinutes())}`;
+  const now = new Date();
+  const sameDay =
+    d.getFullYear() === now.getFullYear() && d.getMonth() === now.getMonth() && d.getDate() === now.getDate();
+  return sameDay ? hms : `${d.toLocaleString("en", { month: "short" })} ${d.getDate()}, ${hms}`;
+}
+
 /** The latest published lemonade version, or undefined when unreachable
  *  (unknown ≠ outdated — never cry wolf). */
 async function latestUpstreamVersion(config: LemonadeConfig): Promise<string | undefined> {
@@ -1066,59 +1171,133 @@ function registerSpecDriftWarnings(pi: ExtensionAPI, configRef: { current: Lemon
 
 // ─── Provider registration ──────────────────────────────────────────────────
 
-function isChatModel(m: LemonadeModel): boolean {
-  return (
-    (m.labels ?? []).includes("chat") ||
-    m.recipe === "llamacpp" ||
-    // Omni collections bundle a chat LLM with server-side tools (image gen,
-    // TTS) and are driven through chat completions like any chat model.
-    m.recipe === "collection.omni"
-  );
+/** One instance's registration-pass probe: catalog + health fetched in
+ *  parallel (null = unreachable). Probes are fetched once per pass — all
+ *  instances at once — and shared by every consumer of that pass (provider
+ *  registration AND the tool-registration classifier hint), so a full
+ *  activation costs one network round per instance even though two
+ *  consumers need the catalog: an unreachable box stalls startup once,
+ *  not once per consumer, and a fleet of instances stalls in parallel,
+ *  not serially. */
+export interface InstanceProbe {
+  view: InstanceView;
+  catalog: LemonadeModel[] | null;
+  health: LemonadeHealth | null;
+  /** Set when catalog/health come from the on-disk outage cache rather
+   *  than a live probe (see withCacheFallback): the epoch ms of the fetch
+   *  the cache entry was written at. */
+  cachedAt?: number;
 }
 
-function toProviderModel(m: LemonadeModel, loaded: Set<string>, providerId = "lemonade") {
-  const labels = m.labels ?? [];
-  const contextWindow = m.max_context_window ?? 128000;
-  return {
-    id: m.id,
-    name: loaded.has(m.id)
-      ? `${m.id} (${providerId}, loaded)`
-      : `${m.id} (${providerId}, on-demand)`,
-    reasoning: labels.includes("reasoning"),
-    input: labels.includes("vision") ? (["text", "image"] as ("text" | "image")[]) : (["text"] as ("text" | "image")[]),
-    cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
-    contextWindow,
-    maxTokens: Math.min(contextWindow, 32768),
-    compat: {
-      supportsDeveloperRole: false,
-      supportsReasoningEffort: false,
-      maxTokensField: "max_tokens" as const,
-    },
-  };
+async function probeInstance(view: InstanceView): Promise<InstanceProbe> {
+  try {
+    const [catalog, health] = await Promise.all([
+      fetchCatalog(view.config),
+      fetchHealth(view.config),
+    ]);
+    if (view.config.cacheOnOutage) {
+      // Fresh truth — persist the snapshot the outage fallback will serve
+      // if a future probe can't reach this box, and clear the down-flag.
+      saveCatalogCacheEntry(view.name, { fetchedAt: Date.now(), catalog, health, unreachableAtLastProbe: false });
+    }
+    return { view, catalog, health };
+  } catch {
+    // Unreachable. What that means for /model depends on the outage cache
+    // (withCacheFallback decides right after) — here we only report the
+    // network truth: the box did not answer within discoveryTimeoutMs.
+    // Durable last-known-state for the hybrid SWR activation path.
+    if (view.config.cacheOnOutage) flagCatalogCacheEntry(view.name, true);
+    console.error(`[pi-lemonade-link] instance "${view.name}" unreachable at ${view.config.baseUrl}`);
+    return { view, catalog: null, health: null };
+  }
+}
+
+/** Probe every configured instance in one parallel pass. */
+function probeInstances(config: LemonadeConfig): Promise<InstanceProbe[]> {
+  return Promise.all(listInstances(config).map(probeInstance));
+}
+
+/** Outage fallback (cacheOnOutage): an unreachable instance serves its last
+ *  successfully cached catalog — provenance-marked "(cached <stamp>)" at
+ *  registration time — instead of listing nothing. Live truth always wins:
+ *  reachable instances pass through untouched, and a box with no cached
+ *  snapshot yet still lists nothing (no cache, nothing honest to serve).
+ *  Idempotent: probes that already carry a cached catalog pass through. */
+function withCacheFallback(config: LemonadeConfig, probes: InstanceProbe[]): InstanceProbe[] {
+  if (!config.cacheOnOutage) return probes;
+  const cache = loadCatalogCache();
+  return probes.map((p) => {
+    if (p.catalog) return p;
+    const entry = cache.instances[p.view.name];
+    if (!entry || !entry.catalog.length) return p;
+    return { ...p, catalog: entry.catalog, health: entry.health, cachedAt: entry.fetchedAt };
+  });
 }
 
-async function registerInstanceProvider(pi: ExtensionAPI, view: InstanceView): Promise<number> {
+/** Hybrid stale-while-revalidate probe pass (cacheOnOutage): instances whose
+ *  LAST probe failed (durable flag, see flagCatalogCacheEntry) serve their
+ *  cached catalog instantly — no probe wait, no discoveryTimeoutMs stall —
+  *  while the rest probe live in one parallel pass (failures still get the
+ *  withCacheFallback safety net, so the transition INTO an outage is the
+ *  only stall left). The caller reconciles instant-served instances with a
+ *  background probe pass. Without the flag (or without a snapshot)
+ *  everything probes live: probe-first is both the honest and the fast path
+ *  when the box was up last time. */
+async function instantOrLiveProbes(config: LemonadeConfig): Promise<InstanceProbe[]> {
+  if (!config.cacheOnOutage) return await probeInstances(config);
+  const cache = loadCatalogCache();
+  const instant: InstanceProbe[] = [];
+  const live: InstanceView[] = [];
+  for (const view of listInstances(config)) {
+    const entry = cache.instances[view.name];
+    if (entry && entry.catalog.length && entry.unreachableAtLastProbe) {
+      instant.push({ view, catalog: entry.catalog, health: entry.health, cachedAt: entry.fetchedAt });
+    } else {
+      live.push(view);
+    }
+  }
+  if (!instant.length) return withCacheFallback(config, await probeInstances(config));
+  return withCacheFallback(config, [...instant, ...(await Promise.all(live.map(probeInstance)))]);
+}
+
+/** Instances whose current /model listing is served from the outage cache
+ *  (probe failed, cache served): name → when it was cached + the listed
+ *  ids. Single source of truth: every registration pass (registerInstanceProvider
+ *  records; registerLemonadeProvider resets) — same discipline as
+ *  serverVersionMismatches. Drives the session-start warning and the
+ *  background re-sync's recovery diff. */
+const cachedServing = new Map<string, { fetchedAt: number; ids: string[] }>();
+
+/** Register one instance's provider from a probe that already ran — no
+ *  network here; the probe pass owns all registration-time I/O. */
+function registerInstanceProvider(pi: ExtensionAPI, probe: InstanceProbe): number {
+  const view = probe.view;
   const config = view.config;
   const providerId = instanceProviderId(view);
-  let catalog: LemonadeModel[];
-  let loaded = new Set<string>();
-  try {
-    const [catalogRes, health] = await Promise.all([
-      fetchCatalog(config),
-      fetchHealth(config),
-    ]);
-    catalog = catalogRes;
-    loaded = new Set((health?.all_models_loaded ?? []).map((m) => m.model_name));
-    recordServerVersion(config, view.name, health?.version);
-  } catch {
-    // Unreachable => contribute nothing to /model. No placeholders: an
-    // unreachable model is not listable. Sessions pinned to a lemonade model
-    // will hard-fail model resolution until the box is reachable again —
-    // the accepted tradeoff of "fully automated, fully honest" listings.
-    catalog = [];
+  const catalog = probe.catalog ?? [];
+  const loaded = new Set((probe.health?.all_models_loaded ?? []).map((m) => m.model_name));
+  const cachedStamp = probe.cachedAt !== undefined ? formatCacheStamp(probe.cachedAt) : undefined;
+  if (cachedStamp) {
+    // Outage-cache serving: the listing is a provenance-marked snapshot, not
+    // live truth — record it so the session-start warning and the background
+    // re-sync know this instance is listed from cache.
     console.error(
-      `[pi-lemonade-link] instance "${view.name}" unreachable at ${config.baseUrl}; registering no models`
+      `[pi-lemonade-link] instance "${view.name}" unreachable — serving cached catalog from ${cachedStamp}`
     );
+    cachedServing.set(view.name, { fetchedAt: probe.cachedAt!, ids: catalog.map((m) => m.id) });
+    recordServerVersion(config, view.name, probe.health?.version); // cached version — self-heals on re-sync
+  } else if (probe.catalog) {
+    cachedServing.delete(view.name);
+    recordServerVersion(config, view.name, probe.health?.version);
+  } else {
+    // Unreachable with no cache to serve: contribute nothing to /model. No
+    // placeholders: an unreachable model is not listable. Sessions pinned to
+    // a lemonade model will hard-fail model resolution until the box is
+    // reachable again — the honest tradeoff when there is no cached truth.
+    console.error(
+      `[pi-lemonade-link] instance "${view.name}" has no cached catalog — registering no models`
+    );
+    cachedServing.delete(view.name);
   }
 
   const advertised = config.chatOnly ? catalog.filter(isChatModel) : catalog;
@@ -1135,12 +1314,45 @@ async function registerInstanceProvider(pi: ExtensionAPI, view: InstanceView): P
     apiKey: config.apiKey || "lemonade",
     authHeader: true,
     api: "openai-completions",
-    models: advertised.map((m) => toProviderModel(m, loaded, providerId)),
+    models: advertised.map((m) => toProviderModel(m, loaded, providerId, cachedStamp)),
   });
 
   return advertised.length;
 }
 
+function isChatModel(m: LemonadeModel): boolean {
+  return (
+    (m.labels ?? []).includes("chat") ||
+    m.recipe === "llamacpp" ||
+    // Omni collections bundle a chat LLM with server-side tools (image gen,
+    // TTS) and are driven through chat completions like any chat model.
+    m.recipe === "collection.omni"
+  );
+}
+
+function toProviderModel(m: LemonadeModel, loaded: Set<string>, providerId = "lemonade", cachedStamp?: string) {
+  const labels = m.labels ?? [];
+  const contextWindow = m.max_context_window ?? 128000;
+  return {
+    id: m.id,
+    name: cachedStamp
+      ? `${m.id} (${providerId}, cached ${cachedStamp})`
+      : loaded.has(m.id)
+        ? `${m.id} (${providerId}, loaded)`
+        : `${m.id} (${providerId}, on-demand)`,
+    reasoning: labels.includes("reasoning"),
+    input: labels.includes("vision") ? (["text", "image"] as ("text" | "image")[]) : (["text"] as ("text" | "image")[]),
+    cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+    contextWindow,
+    maxTokens: Math.min(contextWindow, 32768),
+    compat: {
+      supportsDeveloperRole: false,
+      supportsReasoningEffort: false,
+      maxTokensField: "max_tokens" as const,
+    },
+  };
+}
+
 /** Provider ids used by past versions of this extension. unregisterProvider
  *  only works within the running pi process, and ids changed over the
  *  extension's evolution, so a long-lived session (or /reload after an
@@ -1150,22 +1362,114 @@ async function registerInstanceProvider(pi: ExtensionAPI, view: InstanceView): P
 const LEGACY_PROVIDER_IDS = ["lemonade", "lemonade-undefined"];
 
 /** Register (or re-register) every configured instance. Each becomes its own
- *  pi provider under the uniform lemonade-<name> id. */
-async function registerLemonadeProvider(pi: ExtensionAPI, config: LemonadeConfig): Promise<number> {
+ *  pi provider under the uniform lemonade-<name> id. Probes are shared per
+ *  pass: pass the caller's probeInstances() result to reuse the network
+ *  round (activation does exactly that — one pass feeds both provider and
+ *  tool registration); omitted, a fresh parallel pass runs. */
+async function registerLemonadeProvider(
+  pi: ExtensionAPI,
+  config: LemonadeConfig,
+  probes?: InstanceProbe[]
+): Promise<number> {
   serverVersionMismatches.length = 0; // (re)collected per registration pass
   serverVersionsSeen.clear();
+  cachedServing.clear();
   for (const staleId of LEGACY_PROVIDER_IDS) {
     try {
       pi.unregisterProvider(staleId);
     } catch { /* never registered; fine */ }
   }
   let total = 0;
-  for (const view of listInstances(config)) {
-    total += await registerInstanceProvider(pi, view);
+  for (const probe of probes ?? withCacheFallback(config, await probeInstances(config))) {
+    total += registerInstanceProvider(pi, probe);
   }
   return total;
 }
 
+// ─── Outage re-sync (cacheOnOutage) ─────────────────────────────────────
+
+let resyncBusy = false;
+let lastResyncProbeAt = 0;
+
+/** Fire-and-forget outage re-sync: probe all instances and re-register.
+ *  Throttled to one probe pass per discoveryTimeoutMs window (turn_start
+ *  fires once per turn — agentic loops fire it rapidly) and a no-op while
+ *  every instance is live (cachedServing empty). Recovery is announced
+ *  only when the live catalog differs from the cached one — state-steady
+ *  means silent. force (the setup menu's Refresh connection) bypasses both
+ *  guards and always re-registers. Returns the registered model count
+ *  (−1 when a pass is already in flight). */
+async function resyncFromOutage(
+  pi: ExtensionAPI,
+  configRef: { current: LemonadeConfig },
+  notify?: (message: string) => void,
+  force = false
+): Promise<number> {
+  if (resyncBusy) return -1;
+  if (!force) {
+    if (cachedServing.size === 0) return 0;
+    if (Date.now() - lastResyncProbeAt < configRef.current.discoveryTimeoutMs) return 0;
+  }
+  resyncBusy = true;
+  lastResyncProbeAt = Date.now();
+  try {
+    const cachedBefore = new Map(cachedServing);
+    const probes = await probeInstances(configRef.current);
+    const registered = await registerLemonadeProvider(
+      pi,
+      configRef.current,
+      withCacheFallback(configRef.current, probes)
+    );
+    // Recovery notice — only for instances that answered, and only when
+    // their live catalog actually differs from the served snapshot.
+    for (const [name, before] of cachedBefore) {
+      const recovered = probes.find((p) => p.view.name === name && p.catalog && p.cachedAt === undefined);
+      if (!recovered?.catalog) continue; // still unreachable (or still cache-served)
+      const beforeIds = new Set(before.ids);
+      const afterIds = new Set(recovered.catalog.map((m) => m.id));
+      const added = recovered.catalog.filter((m) => !beforeIds.has(m.id)).map((m) => m.id);
+      const removed = before.ids.filter((id) => !afterIds.has(id));
+      if (!added.length && !removed.length) continue;
+      notify?.(
+        `pi-lemonade-link: instance "${name}" reachable again — re-synced: ` +
+          `${added.length ? `+${added.join(", ")}` : ""}${added.length && removed.length ? "; " : ""}${removed.length ? `−${removed.join(", ")}` : ""}`
+      );
+    }
+    return registered;
+  } finally {
+    resyncBusy = false;
+  }
+}
+
+/** Event wiring for the outage re-sync: prompt submit (turn_start), model
+ *  selection, and session start/resume each trigger a throttled,
+ *  cached-state-only probe pass (never blocking the turn). The status bar's
+ *  own health poll is the free continuous detector (see registerLemonadeBar's
+ *  tick). The session-start handler also warns, one line per outage-cached
+ *  instance, while the state persists — same shape as the tier-1 version
+ *  mismatch warning. */
+function registerOutageResync(pi: ExtensionAPI, configRef: { current: LemonadeConfig }): void {
+  type ExtCtx = import("@earendil-works/pi-coding-agent").ExtensionContext;
+  const trigger = (ctx: ExtCtx) => {
+    if (cachedServing.size === 0) return; // every instance live — nothing to re-sync
+    void resyncFromOutage(pi, configRef, ctx.hasUI ? (m) => ctx.ui.notify(m, "info") : undefined);
+  };
+  pi.on("turn_start", (_e, ctx) => trigger(ctx as ExtCtx));
+  pi.on("model_select", (_e, ctx) => trigger(ctx as ExtCtx));
+  pi.on("session_start", (_e, ctx) => {
+    trigger(ctx as ExtCtx);
+    if (!(ctx as ExtCtx).hasUI) return;
+    for (const [instance, served] of cachedServing) {
+      (ctx as ExtCtx).ui.notify(
+        `pi-lemonade-link: instance "${instance}" is unreachable — /model lists its cached catalog from ` +
+          `${formatCacheStamp(served.fetchedAt)}; requests to it will fail until the box answers. ` +
+          `It re-syncs automatically on the next prompt, model selection, or session start. cacheOnOutage: false disables.`,
+        "warning"
+      );
+    }
+  });
+}
+
 // ─── Multimodal agent tools ──────────────────────────────────────────────────
 
 /** Combine an operation timeout with the caller's cancel signal (if any),
@@ -1427,7 +1731,11 @@ function runFfmpeg(args: string[], timeoutMs: number): Promise<void> {
   });
 }
 
-async function registerAgentTools(pi: ExtensionAPI, gconfig: LemonadeConfig): Promise<void> {
+async function registerAgentTools(
+  pi: ExtensionAPI,
+  gconfig: LemonadeConfig,
+  probes?: InstanceProbe[]
+): Promise<void> {
   // Harness-level self-awareness: these lines are merged into the Guidelines
   // section of every chat's system prompt, so even a virgin chat knows where
   // its lemonade machinery lives without having to hunt for it. (pi dedupes
@@ -1456,19 +1764,18 @@ async function registerAgentTools(pi: ExtensionAPI, gconfig: LemonadeConfig): Pr
   {
     const ids = new Set<string>();
     const pulled = new Set<string>();
-    await Promise.all(
-      listInstances(gconfig).map(async (v) => {
-        try {
-          const catalog = await fetchCatalog(v.config);
-          for (const m of catalog.filter((m) => m.recipe === "onnxruntime")) {
-            ids.add(m.id);
-            if (m.downloaded !== false) pulled.add(m.id);
-          }
-        } catch {
-          // instance unreachable at registration time — hint stays generic for it
-        }
-      })
-    );
+    // Share the activation-pass probes when the caller has them (provider
+    // registration just fetched the same catalogs); otherwise one fresh
+    // parallel pass — never a serial per-instance round.
+    for (const probe of probes ?? withCacheFallback(gconfig, await probeInstances(gconfig))) {
+      if (!probe.catalog) {
+        continue; // instance unreachable at registration time — hint stays generic for it
+      }
+      for (const m of probe.catalog.filter((m) => m.recipe === "onnxruntime")) {
+        ids.add(m.id);
+        if (m.downloaded !== false) pulled.add(m.id);
+      }
+    }
     if (ids.size) {
       classifierHint =
         `Classifiers on this server: ${[...ids].join(", ")}. ` +
@@ -1886,6 +2193,7 @@ function registerLemonadeBar(pi: ExtensionAPI, configRef: { current: LemonadeCon
   const metrics: BarMetrics = { toksPerSec: null, cachePct: null, busy: null, cpuPct: null, gpuPct: null, npuPct: null, npuSupported: null, vramGb: null };
   let activeInstance: string | null = null;
   let timer: ReturnType<typeof setInterval> | undefined;
+  let barCtx: ExtCtx | undefined; // ctx of the installed row — for re-sync notices
   let tuiRef: { requestRender(): void } | undefined;
   let msgStartAt: number | null = null;
 
@@ -1947,6 +2255,13 @@ function registerLemonadeBar(pi: ExtensionAPI, configRef: { current: LemonadeCon
         metrics.npuSupported = false; // box reported system-stats with no NPU
       }
       if (typeof sys?.vram_gb === "number") metrics.vramGb = sys.vram_gb;
+      // Free liveness detector: this poll already proves the box answers —
+      // while its /model listing is served from the outage cache, trigger
+      // the throttled re-sync (throttle/in-flight guards live inside
+      // resyncFromOutage; the notice goes through barCtx while installed).
+      if (cachedServing.has(instance)) {
+        void resyncFromOutage(pi, configRef, (m) => barCtx?.ui.notify(m, "info"));
+      }
     }
     tuiRef?.requestRender();
   };
@@ -1954,6 +2269,7 @@ function registerLemonadeBar(pi: ExtensionAPI, configRef: { current: LemonadeCon
   const teardown = (ctx: ExtCtx) => {
     stopTimer();
     tuiRef = undefined;
+    barCtx = undefined;
     activeInstance = null;
     resetLocal();
     try {
@@ -1967,6 +2283,7 @@ function registerLemonadeBar(pi: ExtensionAPI, configRef: { current: LemonadeCon
     stopTimer();
     resetLocal();
     activeInstance = instance;
+    barCtx = ctx;
     ctx.ui.setWidget(
       KEY,
       (tui, theme) => {
@@ -3001,11 +3318,88 @@ function registerSetupCommand(pi: ExtensionAPI, configRef: { current: LemonadeCo
           { value: "transcription", label: "Transcription settings", description: "Whisper model, paths, smoke test" },
           { value: "logs", label: "Live server logs", description: "Stream the server's log over websocket" },
           { value: "spec", label: "Spec & version check", description: "Docs drift + live server-version drift — status, re-check, re-sync, toggles" },
+          { value: "refresh", label: "Refresh connection", description: "Force a re-probe of every instance now and re-register (bypasses the outage-resync throttle)" },
+          { value: "cache", label: "Catalog cache", description: `Outage listings (cacheOnOutage: ${config.cacheOnOutage ? "on" : "off"}) — cached catalog snapshots: status, toggle, clear` },
           { value: "instance", label: "Switch instance", description: `Current: ${activeInstance} — pick which box status/models/logs act on` },
           { value: "exit", label: "Exit", description: "Close the setup menu" },
         ]);
         if (choice === null || choice === "exit") break;
 
+        // ── Refresh connection (manual outage re-sync) ──────────────────
+        if (choice === "refresh") {
+          const [ok, msg] = await withStatus(ctx, "Probing instances…", async () => {
+            const n = await resyncFromOutage(pi, configRef, undefined, true);
+            if (n < 0) return "A re-sync is already in flight — try again in a moment.";
+            return cachedServing.size
+              ? `Refreshed — ${n} model(s) registered; ${cachedServing.size} instance(s) unreachable, serving cached catalogs.`
+              : `Refreshed — ${n} model(s) registered.`;
+          });
+          ctx.ui.notify(msg, ok ? "info" : "error");
+          continue;
+        }
+
+        // ── Catalog cache (outage listings) ─────────────────────
+        if (choice === "cache") {
+          for (;;) {
+            const config = configRef.current;
+            const cache = loadCatalogCache();
+            const entries = Object.entries(cache.instances).sort(([a], [b]) => a.localeCompare(b));
+            const state = !config.cacheOnOutage
+              ? "disabled (cacheOnOutage: false) — catalogs are neither cached nor served"
+              : entries.length
+                ? entries.map(([name, e]) => `${name}: ${e.catalog.length} model(s) cached ${formatCacheStamp(e.fetchedAt)}`).join(" · ")
+                : "enabled — no snapshots yet (written on every successful probe)";
+            const sub = await menu(ctx, "Catalog Cache — Outage Listings", [
+              { value: "status", label: "Status", description: state },
+              {
+                value: "toggle",
+                label: `Toggle outage cache: ${config.cacheOnOutage ? "on → off" : "off → on"}`,
+                description: `cacheOnOutage (currently ${config.cacheOnOutage ? "on" : "off"}) — when a probe fails, list the instance's last good catalog, provenance-marked "(cached …)"`,
+              },
+              {
+                value: "clear",
+                label: "Clear cached catalogs",
+                description: "Drop the on-disk snapshots — unreachable instances then list nothing",
+              },
+              { value: "back", label: "Back", description: "" },
+            ]);
+            if (sub === null || sub === "back") break;
+
+            if (sub === "toggle") {
+              config.cacheOnOutage = !config.cacheOnOutage;
+              saveConfig(config);
+              ctx.ui.notify(
+                `Outage cache ${config.cacheOnOutage ? "on" : "off"} — takes effect at the next registration (Refresh connection applies it now).`,
+                "info"
+              );
+              continue;
+            }
+            if (sub === "clear") {
+              clearCatalogCache();
+              ctx.ui.notify(
+                "Cached catalogs cleared — unreachable instances will list nothing from the next registration on (Refresh connection applies it now).",
+                "info"
+              );
+              continue;
+            }
+            if (sub === "status") {
+              await textView(ctx, "Catalog Cache — Status", [
+                `cacheOnOutage: ${config.cacheOnOutage ? "on" : "off"}`,
+                `Snapshot path: ${catalogCachePath()}`,
+                "",
+                entries.length ? "Cached catalogs on disk:" : "No cached catalogs on disk.",
+                ...entries.map(([name, e]) => `  ${name} — ${e.catalog.length} model(s), fetched ${formatCacheStamp(e.fetchedAt)}${e.unreachableAtLastProbe ? " — LAST PROBE FAILED (instant-served at startup)" : ""}`),
+                "",
+                "Instances currently listed from cache in /model:",
+                ...(cachedServing.size
+                  ? [...cachedServing].map(([name, s]) => `  ${name} — serving cache from ${formatCacheStamp(s.fetchedAt)}`)
+                  : ["  (none — every instance is live or lists nothing)"]),
+              ]);
+            }
+          }
+          continue;
+        }
+
         // ── Spec & version check ───────────────────────────────────────
         if (choice === "spec") {
           for (;;) {
@@ -3787,11 +4181,33 @@ export default async function piLemonadeLink(pi: ExtensionAPI): Promise<void> {
   // merged from DEFAULT_CONFIG in memory; /lemonade-setup persists the full
   // normalized object when the user actually changes something.
 
-  await registerLemonadeProvider(pi, configRef.current);
-  await registerAgentTools(pi, configRef.current);
+  // One shared parallel probe pass feeds both consumers: an unreachable or
+  // slow box stalls startup once (not once per consumer, not once per
+  // instance) — bounded by a single discoveryTimeoutMs window. With
+  // cacheOnOutage, instances the probe can't reach are listed from their
+  // last good cached catalog, provenance-marked — see withCacheFallback.
+  //
+  // Hybrid SWR: instances whose LAST probe failed (durable flag) are served
+  // from cache instantly — zero probe wait — and reconciled by a background
+  // probe pass; instances last known reachable still probe first, so a
+  // healthy box never flashes stale data. Net: the down-box stall only ever
+  // bites once per outage, at the transition into it.
+  const probes = await instantOrLiveProbes(configRef.current);
+  await registerLemonadeProvider(pi, configRef.current, probes);
+  await registerAgentTools(pi, configRef.current, probes);
+  lastResyncProbeAt = Date.now(); // the activation pass IS a probe pass — throttle the session_start re-sync trigger
+  if (probes.some((p) => p.cachedAt !== undefined)) {
+    // Background reconcile for instant-served instances: probes everything
+    // again (cheap for instances that just answered) and re-registers with
+    // live truth when the box is back. Diff notices are silent here —
+    // activation has no ui ctx, and the session_start cached warning has
+    // already told the user the state; the suffix flips are visible in /model.
+    void resyncFromOutage(pi, configRef, undefined, true);
+  }
   registerLemonadeBar(pi, configRef);
   registerSetupCommand(pi, configRef);
   registerSpecDriftWarnings(pi, configRef);
+  registerOutageResync(pi, configRef);
 
   // Fresh install (or a hand-emptied config): nothing is registered yet —
   // point the user at the wizard instead of leaving them wondering why

package/lemonade.example.json

@@ -51,6 +51,14 @@
   // as pi providers.
   "chatOnly": true,
 
+  // Outage cache: when a probe can't reach an instance, register its last
+  // good catalog (persisted on every successful probe) instead of nothing —
+  // provenance-marked "(cached <time>)" in /model, warned at session start,
+  // and re-synced with a diff notice once the box answers again. Once an
+  // instance is known down (durable flag), the next startup serves the cache
+  // instantly and reconciles in the background — see instantOrLiveProbes.
+  "cacheOnOutage": false,
+
   // --- Model defaults (empty string = auto-pick first in catalog / tool default) ---
   "defaultTranscriptionModel": "Whisper-Large-v3",
   "defaultImageModel": "",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-lemonade-link",
-  "version": "1.0.2",
+  "version": "1.1.0",
   "description": "Unified pi extension for a self-hosted Lemonade server: dynamic chat-model discovery, multimodal agent tools (transcription, image gen/edit/upscale, TTS, audio, 3D mesh), a live /lemonade-setup management TUI, and a lemonade-only below-editor status bar (tok/s, cache hits, busy/queue, VRAM).",
   "keywords": [
     "pi-package",