pi-mega-compact 0.16.1 → 0.16.3

package/dist/src/httpEmbedder.js

@@ -24,9 +24,101 @@
  * blocks without that deadlock. Only used when this embedder is selected; the
  * default TrigramEmbedder path stays pure-sync, zero-network, zero-native.
  */
-import { l2Normalize } from "./embedder.js";
+import { l2Normalize, TrigramEmbedder } from "./embedder.js";
+import { Logger } from "./log.js";
 import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: localhost-only user-spawned embedding server (BYO backend, never remote)
 import { isIP } from "node:net"; // guardrails-allow PREVENT-PI-004: localhost-only loopback address validation in BYO embedding server
+// ── Oversized-input chunking + graceful fallback (BowTiedDevil 500 report) ────
+//
+// The embedding server enforces a physical batch size (llama.cpp n_ctx_slot,
+// commonly 2048 tokens); a single over-large prompt is rejected with HTTP 500
+// ("input is too large to process"). Previously embed() sent the WHOLE region
+// as one prompt and threw on the 500, propagating an unhandled rejection up
+// through VectorStore.addCheckpoint → engine and crashing the checkpoint write.
+//
+// Two layered defenses now live inside embed():
+//   1. CHUNKING — text whose estimated token count exceeds the configured batch
+//      limit is split into <=limit chunks (paragraph/sentence/word boundaries,
+//      never mid-word when avoidable), each chunk embedded, and the per-chunk
+//      vectors mean-pooled (weighted by chunk tokens) + L2-renormalized. This
+//      changes the vector for oversized docs — which previously crashed — and
+//      is the standard pooling approximation for long inputs.
+//   2. FALLBACK — if the server is unreachable / returns an error / returns an
+//      unparseable body, embed() catches and falls back to the local
+//      TrigramEmbedder (logging one structured warn, never the text), so a
+//      server outage degrades semantic dedup instead of crashing the agent loop
+//      (repo non-fatal-store / fails-closed convention).
+//
+// Token count is ESTIMATED, not tokenized: chars / MEGACOMPACT_EMBEDDING_CHARS_PER_TOKEN
+// (default 4, a conservative English/code average). Both knobs are env-tunable.
+/** Default physical batch the typical local server accepts (llama.cpp n_ctx_slot). */
+const DEFAULT_BATCH_TOKENS = 2048;
+/** Conservative chars-per-token average for English + code. */
+const DEFAULT_CHARS_PER_TOKEN = 4;
+function envInt(name, fallback) {
+    const raw = process.env[name];
+    if (raw === undefined)
+        return fallback;
+    const n = Number(raw);
+    return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
+}
+/** Estimated token count of `text` (chars / charsPerToken, rounded up). Exported for tests. */
+export function estimateTokens(text, charsPerToken) {
+    return Math.ceil(text.length / charsPerToken);
+}
+/**
+ * Split `text` into chunks each estimated <= maxTokens. Prefers paragraph, then
+ * sentence, then word boundaries; falls back to a hard slice only when a single
+ * word/run still exceeds the limit (never splits a word unless unavoidable).
+ */
+export function chunkText(text, maxTokens, charsPerToken) {
+    const maxChars = maxTokens * charsPerToken;
+    if (text.length <= maxChars)
+        return [text];
+    const chunks = [];
+    let start = 0;
+    while (start < text.length) {
+        let end = Math.min(start + maxChars, text.length);
+        if (end < text.length) {
+            // Back off to the last nice boundary within the window: paragraph, then
+            // sentence, then whitespace. Require the boundary in the back half so we
+            // don't pathologically emit tiny chunks.
+            const windowStart = start + Math.floor(maxChars / 2);
+            const para = text.lastIndexOf("\n\n", end);
+            const sentence = text.lastIndexOf(". ", end);
+            const space = text.lastIndexOf(" ", end);
+            if (para >= windowStart)
+                end = para + 2;
+            else if (sentence >= windowStart)
+                end = sentence + 2;
+            else if (space >= windowStart)
+                end = space + 1;
+            // else: no good boundary in the back half — hard-slice at maxChars.
+        }
+        const piece = text.slice(start, end);
+        if (piece.trim().length > 0)
+            chunks.push(piece);
+        start = end;
+    }
+    return chunks;
+}
+/** Weighted mean-pool of equal-dim vectors (weights = chunk token estimates), then L2-renormalize. */
+export function meanPool(vectors, weights) {
+    const dim = vectors[0]?.length ?? 0;
+    const acc = new Array(dim).fill(0);
+    let total = 0;
+    for (let i = 0; i < vectors.length; i++) {
+        const w = weights[i] ?? 0;
+        total += w;
+        const v = vectors[i];
+        for (let d = 0; d < dim; d++)
+            acc[d] += v[d] * w;
+    }
+    if (total > 0)
+        for (let d = 0; d < dim; d++)
+            acc[d] /= total;
+    return l2Normalize(acc);
+}
 // Inline worker script: resolves a hostname via dns.lookup in a child process
 // (dns.lookup is callback-async; the child has its own event loop). Used to
 // verify that a hostname in the embedding URL resolves to loopback ONLY.
@@ -204,11 +296,19 @@ export class HttpEmbedder {
     apiKey;
     headers;
     resolvedDim;
+    batchTokens;
+    charsPerToken;
+    fallback;
+    logger;
     constructor(opts) {
         this.url = opts.url;
         this.apiKey = opts.apiKey;
         this.headers = opts.headers ?? {};
         this.resolvedDim = opts.dim ?? 0; // resolved after the first embed
+        this.batchTokens = envInt("MEGACOMPACT_EMBEDDING_BATCH_TOKENS", DEFAULT_BATCH_TOKENS);
+        this.charsPerToken = envInt("MEGACOMPACT_EMBEDDING_CHARS_PER_TOKEN", DEFAULT_CHARS_PER_TOKEN);
+        this.fallback = new TrigramEmbedder();
+        this.logger = new Logger();
     }
     get dim() {
         return this.resolvedDim;
@@ -218,7 +318,42 @@ export class HttpEmbedder {
     get chatUrl() {
         return new URL("/api/chat", this.url).href;
     }
+    /**
+     * Embed `text`. Oversized inputs are chunked (each chunk <= the configured
+     * server batch) and mean-pooled; a server failure falls back to the local
+     * TrigramEmbedder. NEVER throws — a misbehaving server must degrade semantic
+     * dedup, not crash the checkpoint write path (VectorStore.addCheckpoint has
+     * no try/catch around embed()).
+     */
     embed(text) {
+        const est = estimateTokens(text, this.charsPerToken);
+        const chunks = est > this.batchTokens ? chunkText(text, this.batchTokens, this.charsPerToken) : [text];
+        try {
+            if (chunks.length === 1) {
+                return this.embedOne(chunks[0]);
+            }
+            const vectors = [];
+            const weights = [];
+            for (const c of chunks) {
+                vectors.push(this.embedOne(c));
+                weights.push(estimateTokens(c, this.charsPerToken));
+            }
+            return meanPool(vectors, weights);
+        }
+        catch (e) {
+            // Graceful fallback — degrade to the local embedder rather than crash.
+            // Never log the text itself (privacy / PREVENT-PI-004).
+            this.logger.warn("embedder_http_fallback", {
+                reason: e instanceof Error ? e.message : String(e),
+                chunks: chunks.length,
+                chars: text.length,
+                estTokens: est,
+            });
+            return this.fallback.embed(text);
+        }
+    }
+    /** Single request/response round-trip for one chunk. Throws on any failure. */
+    embedOne(text) {
         const ollama = isOllamaEndpoint(this.url);
         const body = ollama
             ? JSON.stringify({ model: process.env.MEGACOMPACT_OLLAMA_MODEL || "nomic-embed-text", prompt: text })

package/dist/src/vector-search.js

@@ -41,7 +41,7 @@ function raptorSearchHits(store, sid, query, k, checkpoints) {
         const stateDir = store.stateDir;
         const cfg = store.cfg;
         const embedder = store.embedder;
-        const record = store.record;
+        const record = store.record.bind(store);
         // S25 gate (a): honor shadow mode at SERVE time. When RAPTOR_SHADOW_MODE=true,
         // the tree is built + persisted but NOT merged into recall (transition/eval).
         // Default is live: shadow mode is opt-in, not the default.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.16.1",
+  "version": "0.16.3",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/httpEmbedder.ts

@@ -26,7 +26,8 @@
  */
 
 import type { Embedder, Vector } from "./embedder.js";
-import { l2Normalize } from "./embedder.js";
+import { l2Normalize, TrigramEmbedder } from "./embedder.js";
+import { Logger } from "./log.js";
 import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: localhost-only user-spawned embedding server (BYO backend, never remote)
 import { isIP } from "node:net"; // guardrails-allow PREVENT-PI-004: localhost-only loopback address validation in BYO embedding server
 
@@ -40,6 +41,94 @@ export interface HttpEmbedderOptions {
   dim?: number;
 }
 
+// ── Oversized-input chunking + graceful fallback (BowTiedDevil 500 report) ────
+//
+// The embedding server enforces a physical batch size (llama.cpp n_ctx_slot,
+// commonly 2048 tokens); a single over-large prompt is rejected with HTTP 500
+// ("input is too large to process"). Previously embed() sent the WHOLE region
+// as one prompt and threw on the 500, propagating an unhandled rejection up
+// through VectorStore.addCheckpoint → engine and crashing the checkpoint write.
+//
+// Two layered defenses now live inside embed():
+//   1. CHUNKING — text whose estimated token count exceeds the configured batch
+//      limit is split into <=limit chunks (paragraph/sentence/word boundaries,
+//      never mid-word when avoidable), each chunk embedded, and the per-chunk
+//      vectors mean-pooled (weighted by chunk tokens) + L2-renormalized. This
+//      changes the vector for oversized docs — which previously crashed — and
+//      is the standard pooling approximation for long inputs.
+//   2. FALLBACK — if the server is unreachable / returns an error / returns an
+//      unparseable body, embed() catches and falls back to the local
+//      TrigramEmbedder (logging one structured warn, never the text), so a
+//      server outage degrades semantic dedup instead of crashing the agent loop
+//      (repo non-fatal-store / fails-closed convention).
+//
+// Token count is ESTIMATED, not tokenized: chars / MEGACOMPACT_EMBEDDING_CHARS_PER_TOKEN
+// (default 4, a conservative English/code average). Both knobs are env-tunable.
+
+/** Default physical batch the typical local server accepts (llama.cpp n_ctx_slot). */
+const DEFAULT_BATCH_TOKENS = 2048;
+/** Conservative chars-per-token average for English + code. */
+const DEFAULT_CHARS_PER_TOKEN = 4;
+
+function envInt(name: string, fallback: number): number {
+  const raw = process.env[name];
+  if (raw === undefined) return fallback;
+  const n = Number(raw);
+  return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
+}
+
+/** Estimated token count of `text` (chars / charsPerToken, rounded up). Exported for tests. */
+export function estimateTokens(text: string, charsPerToken: number): number {
+  return Math.ceil(text.length / charsPerToken);
+}
+
+/**
+ * Split `text` into chunks each estimated <= maxTokens. Prefers paragraph, then
+ * sentence, then word boundaries; falls back to a hard slice only when a single
+ * word/run still exceeds the limit (never splits a word unless unavoidable).
+ */
+export function chunkText(text: string, maxTokens: number, charsPerToken: number): string[] {
+  const maxChars = maxTokens * charsPerToken;
+  if (text.length <= maxChars) return [text];
+  const chunks: string[] = [];
+  let start = 0;
+  while (start < text.length) {
+    let end = Math.min(start + maxChars, text.length);
+    if (end < text.length) {
+      // Back off to the last nice boundary within the window: paragraph, then
+      // sentence, then whitespace. Require the boundary in the back half so we
+      // don't pathologically emit tiny chunks.
+      const windowStart = start + Math.floor(maxChars / 2);
+      const para = text.lastIndexOf("\n\n", end);
+      const sentence = text.lastIndexOf(". ", end);
+      const space = text.lastIndexOf(" ", end);
+      if (para >= windowStart) end = para + 2;
+      else if (sentence >= windowStart) end = sentence + 2;
+      else if (space >= windowStart) end = space + 1;
+      // else: no good boundary in the back half — hard-slice at maxChars.
+    }
+    const piece = text.slice(start, end);
+    if (piece.trim().length > 0) chunks.push(piece);
+    start = end;
+  }
+  return chunks;
+}
+
+/** Weighted mean-pool of equal-dim vectors (weights = chunk token estimates), then L2-renormalize. */
+export function meanPool(vectors: readonly Vector[], weights: readonly number[]): Vector {
+  const dim = vectors[0]?.length ?? 0;
+  const acc = new Array<number>(dim).fill(0);
+  let total = 0;
+  for (let i = 0; i < vectors.length; i++) {
+    const w = weights[i] ?? 0;
+    total += w;
+    const v = vectors[i];
+    for (let d = 0; d < dim; d++) acc[d] += v[d] * w;
+  }
+  if (total > 0) for (let d = 0; d < dim; d++) acc[d] /= total;
+  return l2Normalize(acc);
+}
+
 // Inline worker script: resolves a hostname via dns.lookup in a child process
 // (dns.lookup is callback-async; the child has its own event loop). Used to
 // verify that a hostname in the embedding URL resolves to loopback ONLY.
@@ -217,12 +306,20 @@ export class HttpEmbedder implements Embedder {
   private readonly apiKey?: string;
   private readonly headers: Record<string, string>;
   private resolvedDim: number;
+  private readonly batchTokens: number;
+  private readonly charsPerToken: number;
+  private readonly fallback: TrigramEmbedder;
+  private readonly logger: Logger;
 
   constructor(opts: HttpEmbedderOptions) {
     this.url = opts.url;
     this.apiKey = opts.apiKey;
     this.headers = opts.headers ?? {};
     this.resolvedDim = opts.dim ?? 0; // resolved after the first embed
+    this.batchTokens = envInt("MEGACOMPACT_EMBEDDING_BATCH_TOKENS", DEFAULT_BATCH_TOKENS);
+    this.charsPerToken = envInt("MEGACOMPACT_EMBEDDING_CHARS_PER_TOKEN", DEFAULT_CHARS_PER_TOKEN);
+    this.fallback = new TrigramEmbedder();
+    this.logger = new Logger();
   }
 
   get dim(): number {
@@ -235,7 +332,43 @@ export class HttpEmbedder implements Embedder {
     return new URL("/api/chat", this.url).href;
   }
 
+  /**
+   * Embed `text`. Oversized inputs are chunked (each chunk <= the configured
+   * server batch) and mean-pooled; a server failure falls back to the local
+   * TrigramEmbedder. NEVER throws — a misbehaving server must degrade semantic
+   * dedup, not crash the checkpoint write path (VectorStore.addCheckpoint has
+   * no try/catch around embed()).
+   */
   embed(text: string): Vector {
+    const est = estimateTokens(text, this.charsPerToken);
+    const chunks =
+      est > this.batchTokens ? chunkText(text, this.batchTokens, this.charsPerToken) : [text];
+    try {
+      if (chunks.length === 1) {
+        return this.embedOne(chunks[0]);
+      }
+      const vectors: Vector[] = [];
+      const weights: number[] = [];
+      for (const c of chunks) {
+        vectors.push(this.embedOne(c));
+        weights.push(estimateTokens(c, this.charsPerToken));
+      }
+      return meanPool(vectors, weights);
+    } catch (e) {
+      // Graceful fallback — degrade to the local embedder rather than crash.
+      // Never log the text itself (privacy / PREVENT-PI-004).
+      this.logger.warn("embedder_http_fallback", {
+        reason: e instanceof Error ? e.message : String(e),
+        chunks: chunks.length,
+        chars: text.length,
+        estTokens: est,
+      });
+      return this.fallback.embed(text);
+    }
+  }
+
+  /** Single request/response round-trip for one chunk. Throws on any failure. */
+  private embedOne(text: string): Vector {
     const ollama = isOllamaEndpoint(this.url);
     const body = ollama
       ? JSON.stringify({ model: process.env.MEGACOMPACT_OLLAMA_MODEL || "nomic-embed-text", prompt: text })

package/src/vector-search.ts

@@ -62,7 +62,7 @@ function raptorSearchHits(
 		const stateDir = store.stateDir;
 		const cfg = store.cfg;
 		const embedder = store.embedder;
-		const record = store.record;
+		const record = store.record.bind(store);
 
 		// S25 gate (a): honor shadow mode at SERVE time. When RAPTOR_SHADOW_MODE=true,
 		// the tree is built + persisted but NOT merged into recall (transition/eval).