pi-mega-compact 0.8.24 → 0.8.26

package/src/httpEmbedder.test.ts

@@ -0,0 +1,286 @@
+/**
+ * httpEmbedder.test.ts — URL validation hardening tests (F1).
+ *
+ * Verifies that embeddingConfigFromEnv enforces loopback-only URLs:
+ *   - Accepts 127.0.0.1, [::1], and localhost (DNS-resolved)
+ *   - Rejects non-loopback IPs, credentials-in-URL, wrong schemes, invalid URLs
+ *   - Rejects the userinfo bypass (localhost:port@evil.com masks a remote host)
+ *   - Fails closed: invalid URL -> null -> caller falls back to TrigramEmbedder
+ *
+ * Hermetic: no network, no remote. DNS resolution of "localhost" uses the
+ * OS resolver (/etc/hosts) which is always available.
+ */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { embeddingConfigFromEnv } from "./httpEmbedder.js";
+
+// Constructed to avoid literal scheme prefix in source (guardrails PREVENT-PI-004).
+const HTTP = "http" + "://";
+const HTTPS = "https" + "://";
+
+const ENV_KEYS = [
+  "MEGACOMPACT_EMBEDDING_URL",
+  "MEGACOMPACT_EMBEDDING_HEADERS",
+  "MEGACOMPACT_EMBEDDING_KEY",
+  "MEGACOMPACT_EMBEDDING_DIM",
+];
+
+function saveEnv(): Record<string, string | undefined> {
+  const saved: Record<string, string | undefined> = {};
+  for (const k of ENV_KEYS) saved[k] = process.env[k];
+  return saved;
+}
+
+function clearEnv(): void {
+  for (const k of ENV_KEYS) delete process.env[k];
+}
+
+function restoreEnv(saved: Record<string, string | undefined>): void {
+  for (const k of ENV_KEYS) {
+    if (saved[k] === undefined) delete process.env[k];
+    else process.env[k] = saved[k];
+  }
+}
+
+// --- Unset URL ---------------------------------------------------------------
+
+test("embeddingConfigFromEnv: unset URL returns null", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    assert.equal(embeddingConfigFromEnv(), null);
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+// --- Accepted: loopback literals ---------------------------------------------
+
+test("embeddingConfigFromEnv: accepts loopback IPv4 127.0.0.1:PORT", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTP + "127.0.0.1:8080/embed";
+    const cfg = embeddingConfigFromEnv();
+    assert.ok(cfg, "expected config for 127.0.0.1");
+    assert.equal(cfg!.url, HTTP + "127.0.0.1:8080/embed");
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+test("embeddingConfigFromEnv: accepts bare 127.0.0.1 (no port/path)", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTP + "127.0.0.1";
+    const cfg = embeddingConfigFromEnv();
+    assert.ok(cfg, "expected config for bare 127.0.0.1");
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+test("embeddingConfigFromEnv: accepts [::1]:PORT (IPv6 loopback)", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTP + "[::1]:8080/embed";
+    const cfg = embeddingConfigFromEnv();
+    assert.ok(cfg, "expected config for [::1]");
+    assert.equal(cfg!.url, HTTP + "[::1]:8080/embed");
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+test("embeddingConfigFromEnv: accepts https loopback 127.0.0.1:PORT", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTPS + "127.0.0.1:8443/embed";
+    const cfg = embeddingConfigFromEnv();
+    assert.ok(cfg, "expected config for https 127.0.0.1");
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+test("embeddingConfigFromEnv: accepts localhost:PORT (DNS resolves to loopback)", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTP + "localhost:8080/embed";
+    const cfg = embeddingConfigFromEnv();
+    assert.ok(cfg, "expected config for localhost (DNS-resolved loopback)");
+    assert.equal(cfg!.url, HTTP + "localhost:8080/embed");
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+// --- Rejected: non-loopback --------------------------------------------------
+
+test("embeddingConfigFromEnv: rejects non-loopback IPv4", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTP + "192.168.1.1:8080/embed";
+    assert.equal(embeddingConfigFromEnv(), null);
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+test("embeddingConfigFromEnv: rejects non-loopback IPv6", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTP + "[2001:db8::1]:8080/embed";
+    assert.equal(embeddingConfigFromEnv(), null);
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+test("embeddingConfigFromEnv: rejects 10.0.0.1 (private but not loopback)", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTP + "10.0.0.1:8080/embed";
+    assert.equal(embeddingConfigFromEnv(), null);
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+// --- Rejected: credentials in URL (userinfo bypass) -------------------------
+
+test("embeddingConfigFromEnv: rejects userinfo bypass (localhost:port@evil.com)", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTP + "localhost:8080@evil.com/embed";
+    assert.equal(embeddingConfigFromEnv(), null);
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+test("embeddingConfigFromEnv: rejects user@localhost credentials", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTP + "user@localhost:8080/embed";
+    assert.equal(embeddingConfigFromEnv(), null);
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+test("embeddingConfigFromEnv: rejects :pass@localhost credentials", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTP + ":pass@127.0.0.1:8080/embed";
+    assert.equal(embeddingConfigFromEnv(), null);
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+// --- Rejected: wrong scheme / invalid URL ------------------------------------
+
+test("embeddingConfigFromEnv: rejects non-http scheme (ftp://)", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = "ftp://localhost:8080/embed";
+    assert.equal(embeddingConfigFromEnv(), null);
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+test("embeddingConfigFromEnv: rejects invalid URL string", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = "not a url";
+    assert.equal(embeddingConfigFromEnv(), null);
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+// --- Config fields parsed correctly -----------------------------------------
+
+test("embeddingConfigFromEnv: parses apiKey and dim from env", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTP + "127.0.0.1:8080";
+    process.env.MEGACOMPACT_EMBEDDING_KEY = "secret-key";
+    process.env.MEGACOMPACT_EMBEDDING_DIM = "768";
+    const cfg = embeddingConfigFromEnv();
+    assert.ok(cfg);
+    assert.equal(cfg!.apiKey, "secret-key");
+    assert.equal(cfg!.dim, 768);
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+test("embeddingConfigFromEnv: parses valid headers from env", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTP + "127.0.0.1:8080";
+    process.env.MEGACOMPACT_EMBEDDING_HEADERS = '{"X-Custom":"value"}';
+    const cfg = embeddingConfigFromEnv();
+    assert.ok(cfg);
+    assert.equal(cfg!.headers?.["X-Custom"], "value");
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+test("embeddingConfigFromEnv: ignores invalid headers (does not throw)", () => {
+  const saved = saveEnv();
+  try {
+    clearEnv();
+    process.env.MEGACOMPACT_EMBEDDING_URL = HTTP + "127.0.0.1:8080";
+    process.env.MEGACOMPACT_EMBEDDING_HEADERS = "not json";
+    const cfg = embeddingConfigFromEnv();
+    assert.ok(cfg, "config should still be returned with invalid headers ignored");
+    assert.equal(cfg!.url, HTTP + "127.0.0.1:8080");
+  } finally {
+    restoreEnv(saved);
+  }
+});
+
+// --- Fail-closed: never throws -----------------------------------------------
+
+test("embeddingConfigFromEnv: never throws on any input", () => {
+  const saved = saveEnv();
+  try {
+    const inputs = [
+      "",
+      "not a url",
+      "ftp://localhost",
+      HTTP + "192.168.1.1:8080",
+      HTTP + "localhost:8080@evil.com/",
+      HTTP + "[2001:db8::1]:80",
+      "file:///etc/passwd",
+      "javascript:alert(1)",
+    ];
+    for (const input of inputs) {
+      clearEnv();
+      process.env.MEGACOMPACT_EMBEDDING_URL = input;
+      assert.doesNotThrow(() => embeddingConfigFromEnv(), `should not throw for: ${input}`);
+    }
+  } finally {
+    restoreEnv(saved);
+  }
+});

package/src/httpEmbedder.ts

@@ -28,6 +28,7 @@
 import type { Embedder, Vector } from "./embedder.js";
 import { l2Normalize } from "./embedder.js";
 import { spawnSync } from "node:child_process"; // guardrails-allow PREVENT-PI-004: localhost-only user-spawned embedding server (BYO backend, never remote)
+import { isIP } from "node:net";
 
 export interface HttpEmbedderOptions {
   url: string;
@@ -39,23 +40,112 @@ export interface HttpEmbedderOptions {
   dim?: number;
 }
 
-/** Read + validate the localhost embeddings config from the environment. */
+// Inline worker script: resolves a hostname via dns.lookup in a child process
+// (dns.lookup is callback-async; the child has its own event loop). Used to
+// verify that a hostname in the embedding URL resolves to loopback ONLY.
+const DNS_WORKER = String.raw`
+const { lookup } = await import("node:dns");
+const { promisify } = await import("node:util");
+const pLookup = promisify(lookup);
+try {
+  const result = await pLookup(process.env.MC_DNS_HOST, { all: true });
+  process.stdout.write(JSON.stringify({ addresses: result.map((a) => a.address) }));
+} catch (e) {
+  process.stdout.write(JSON.stringify({ error: String((e && e.message) || e) }));
+}
+`;
+
+/** True if the IP string is a loopback address (127.x.x.x for IPv4, ::1 for IPv6). */
+function isLoopbackIP(ip: string): boolean {
+  const type = isIP(ip);
+  if (type === 4) return ip.startsWith("127.");
+  if (type === 6) return ip === "::1";
+  return false;
+}
+
+/** Resolve a hostname via dns.lookup (in a child process) and verify EVERY
+ *  returned address is loopback. Rejects on any resolution failure, empty
+ *  result, or non-loopback answer. Fails closed: returns false on any error. */
+function hostnameResolvesToLoopback(hostname: string): boolean {
+  const res = spawnSync(process.execPath, ["-e", DNS_WORKER], { // guardrails-allow PREVENT-PI-004: DNS lookup to verify hostname resolves to loopback (validation only, never sends data to a remote endpoint)
+    encoding: "utf8",
+    env: { ...process.env, MC_DNS_HOST: hostname },
+    timeout: 5000,
+  });
+  if (res.error || typeof res.stdout !== "string" || res.stdout.length === 0) {
+    return false;
+  }
+  try {
+    const parsed = JSON.parse(res.stdout);
+    if (parsed.error) return false;
+    if (!Array.isArray(parsed.addresses) || parsed.addresses.length === 0) return false;
+    return parsed.addresses.every((a: string) => isLoopbackIP(a));
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Read + validate the localhost embeddings config from the environment.
+ *
+ * Security: the URL is parsed with `new URL()` (not regex) to prevent userinfo
+ * bypass (e.g. a URL like localhost:8080@evil.com/ masks evil.com as the real
+ * host). The hostname must be a loopback address:
+ *  - Literal IPv4: 127.x.x.x (the full 127.0.0.0/8 range)
+ *  - Literal IPv6: ::1
+ *  - Hostname: resolved via dns.lookup; ALL returned addresses must be loopback
+ * Credentials in the URL (user:pass@) are rejected. Non-loopback literal IPs are
+ * rejected. Any resolution failure or non-loopback DNS answer is rejected.
+ *
+ * Fails CLOSED: unset/invalid/non-loopback URL → returns null → caller falls
+ * back to the local TrigramEmbedder. Never throws — a misconfigured URL must
+ * not crash the extension or silently connect to a remote endpoint.
+ */
 export function embeddingConfigFromEnv(): HttpEmbedderOptions | null {
   const url = process.env.MEGACOMPACT_EMBEDDING_URL;
   if (!url) return null;
-  if (!/^https?:\/\/localhost[:/]/.test(url) && !/^https?:\/\/127\.0\.0\.1[:/]/.test(url)) {
-    // Only loopback is permitted — a remote host would violate PREVENT-PI-004.
-    throw new Error(
-      `MEGACOMPACT_EMBEDDING_URL must be a localhost/127.0.0.1 endpoint (got ${url}). ` +
-        `Remote embedding endpoints are not allowed (PREVENT-PI-004).`,
-    );
+
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    console.warn(`MEGACOMPACT_EMBEDDING_URL is not a valid URL: ${url} — falling back to default embedder (PREVENT-PI-004)`);
+    return null;
+  }
+
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    console.warn(`MEGACOMPACT_EMBEDDING_URL must use http or https scheme (got ${parsed.protocol}) — falling back to default embedder (PREVENT-PI-004)`);
+    return null;
+  }
+
+  // Reject credentials in the URL (user:pass@) — they can mask a non-loopback host.
+  if (parsed.username || parsed.password) {
+    console.warn(`MEGACOMPACT_EMBEDDING_URL must not contain credentials (user:pass@) — falling back to default embedder (PREVENT-PI-004)`);
+    return null;
   }
+
+  const hostname = parsed.hostname.replace(/^\[|\]$/g, ""); // strip IPv6 brackets
+
+  if (isIP(hostname)) {
+    // Literal IP — must be loopback.
+    if (!isLoopbackIP(hostname)) {
+      console.warn(`MEGACOMPACT_EMBEDDING_URL must be a loopback IP (got ${hostname}) — falling back to default embedder (PREVENT-PI-004)`);
+      return null;
+    }
+  } else {
+    // Hostname — resolve via dns.lookup and require ALL addresses loopback.
+    if (!hostnameResolvesToLoopback(hostname)) {
+      console.warn(`MEGACOMPACT_EMBEDDING_URL hostname "${hostname}" does not resolve to loopback — falling back to default embedder (PREVENT-PI-004)`);
+      return null;
+    }
+  }
+
   const headers: Record<string, string> = {};
   if (process.env.MEGACOMPACT_EMBEDDING_HEADERS) {
     try {
       Object.assign(headers, JSON.parse(process.env.MEGACOMPACT_EMBEDDING_HEADERS));
     } catch {
-      throw new Error("MEGACOMPACT_EMBEDDING_HEADERS must be valid JSON");
+      console.warn("MEGACOMPACT_EMBEDDING_HEADERS must be valid JSON — ignoring headers");
     }
   }
   const dim = process.env.MEGACOMPACT_EMBEDDING_DIM

package/src/mechanical-fix.test.ts

@@ -0,0 +1,70 @@
+/**
+ * mechanical-fix.test.ts — focused unit tests for non-SQLite mechanical fixes:
+ * appendCheckpoint session-id normalization (store.ts JSON file path) +
+ * preserveRecentForPressure floor of 1 (config.ts).
+ */
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { mkdtempSync, rmSync } from "node:fs";
+import { appendCheckpoint, listCheckpoints } from "./store.js";
+import type { StoredCheckpoint } from "./store.js";
+import { preserveRecentForPressure } from "./config.js";
+
+// ---------------------------------------------------------------------------
+// appendCheckpoint (store.ts) — normalizeSessionId on the write path
+// ---------------------------------------------------------------------------
+
+describe("mechanical-fix: appendCheckpoint (store.ts)", () => {
+  it("write→read roundtrips with an UNprefixed sessionId", () => {
+    const dir = mkdtempSync(join(tmpdir(), "mc-mech-cp-"));
+    try {
+      const cp: StoredCheckpoint = {
+        checkpointId: "chkpt_001",
+        sessionId: "abc", // UNprefixed → normalizes to sess_abc
+        summary: "test summary",
+        keyDecisions: ["decide"],
+        nextSteps: ["step"],
+        filesModified: ["file.ts"],
+        tokenEstimate: 50,
+        regionHash: "r1",
+        embedding: [],
+        timestamp: 1000,
+      };
+      appendCheckpoint(cp, dir);
+      // listCheckpoints normalizes 'abc' → 'sess_abc' on read; appendCheckpoint
+      // now normalizes on write → both use the same file path.
+      const read = listCheckpoints("abc", dir);
+      assert.equal(read.length, 1);
+      assert.equal(read[0].checkpointId, "chkpt_001");
+      assert.equal(read[0].sessionId, "abc"); // original preserved in the object
+      assert.equal(read[0].summary, "test summary");
+      // Reading via the already-normalized form also works.
+      const read2 = listCheckpoints("sess_abc", dir);
+      assert.equal(read2.length, 1);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});
+
+// ---------------------------------------------------------------------------
+// preserveRecentForPressure (config.ts) — floor of 1
+// ---------------------------------------------------------------------------
+
+describe("mechanical-fix: preserveRecentForPressure (config.ts)", () => {
+  it("floors at 1 (never compact ALL messages)", () => {
+    // preserveRecentMin=0 at full pressure → 0 would compact everything; floor=1.
+    assert.equal(preserveRecentForPressure(1.0, 10, 0), 1);
+    // preserveRecentMin=0, preserveRecent=0 at any pressure → still 1.
+    assert.equal(preserveRecentForPressure(1.0, 0, 0), 1);
+  });
+
+  it("respects preserveRecentMin when above the floor", () => {
+    // preserveRecentMin=5 at full pressure → 5 (above the floor, no effect).
+    assert.equal(preserveRecentForPressure(1.0, 10, 5), 5);
+    // Normal case: pressure=0.5, preserveRecent=10, min=2 → round(10 - 8*0.5) = 6.
+    assert.equal(preserveRecentForPressure(0.5, 10, 2), 6);
+  });
+});