pi-mega-compact 0.20.55 → 0.20.57

package/extensions/dashboard-server/api-contracts/index.ts

@@ -357,3 +357,15 @@ export type {
 	CosineFpOverrides,
 	CosineFpReportV1,
 } from "./cosine-fp.js";
+
+// REPO-A — cross-repo corpus status (reader-only)
+export type {
+	RepoCorpusStatus,
+	RepoCorpusRepoV1,
+	RepoCorpusOverlapV1,
+	RepoCorpusManifestV1,
+	RepoCorpusConsentStateV1,
+	RepoCorpusConsentRowV1,
+	RepoCorpusPerRepoStatusV1,
+	RepoCorpusStatusV1,
+} from "./repo-corpus.js";

package/extensions/dashboard-server/api-contracts/repo-corpus.ts

@@ -0,0 +1,76 @@
+/**
+ * api-contracts/repo-corpus.ts — REPO-A cross-repo corpus API contract.
+ *
+ * Reader-only shapes for GET /api/repo-corpus. The corpus manifest is written by
+ * the CLI corpus-builder (scripts/repo-corpus/build.mjs) as a PSEUDONYMOUS
+ * artifact — repo IDs/counts/digests/cross-repo-overlap descriptors only, never
+ * payload content (EVAL-REDACT-002). The canonicalRemote→pseudonym mapping is a
+ * pure in-memory function and never leaves the builder. No `any`.
+ */
+
+/** Derived endpoint status (from vc-status.ts). */
+export type RepoCorpusStatus =
+  | "live"
+  | "awaiting_data"
+  | "deferred"
+  | "structural"
+  | "off";
+
+/** One pseudonymous repo in the corpus manifest. */
+export interface RepoCorpusRepoV1 {
+  readonly repoPseudonym: string;
+  readonly sessions: number;
+  readonly sessionIds: readonly string[];
+  readonly events: number;
+  /** SHA-256 over the repo's session-id slice + count (content-free). */
+  readonly digest: string;
+}
+
+/** A recorded cross-repo session overlap descriptor (IDs only, never text). */
+export interface RepoCorpusOverlapV1 {
+  readonly repoA: string;
+  readonly repoB: string;
+  readonly sharedSessions: number;
+  readonly sharedIds: readonly string[];
+}
+
+/** The corpus manifest the builder emits (pseudonymous). */
+export interface RepoCorpusManifestV1 {
+  readonly schema: "repo-corpus-manifest-v1";
+  readonly ownerPseudonym: string;
+  readonly datasetVersion: string;
+  readonly effectiveSeq: number;
+  readonly totalEvents: number;
+  readonly repos: readonly RepoCorpusRepoV1[];
+  readonly overlaps: readonly RepoCorpusOverlapV1[];
+}
+
+/** Per-repo consent state projection used by the reader route (pseudonymous). */
+export interface RepoCorpusConsentStateV1 {
+  readonly schema: "repo-corpus-consent-state-v1";
+  readonly perRepo: readonly RepoCorpusConsentRowV1[];
+}
+
+/** One repo's consent row (never payload content). */
+export interface RepoCorpusConsentRowV1 {
+  readonly repoPseudonym: string;
+  readonly consentedCrossRepo: boolean;
+  readonly revokedAt?: string;
+}
+
+/** Per-repo status row in RepoCorpusStatusV1. */
+export interface RepoCorpusPerRepoStatusV1 {
+  readonly repoPseudonym: string;
+  readonly sessions: number;
+  readonly consentedCrossRepo: boolean;
+  readonly revokedAt?: string;
+}
+
+/** Response body for GET /api/repo-corpus. */
+export interface RepoCorpusStatusV1 {
+  readonly schema: "repo-corpus-status-v1";
+  readonly corpus: RepoCorpusManifestV1 | null;
+  readonly perRepo: readonly RepoCorpusPerRepoStatusV1[];
+  readonly totalEvents: number;
+  readonly status: RepoCorpusStatus;
+}

package/extensions/dashboard-server/route-dispatch.ts

@@ -64,6 +64,7 @@ import {
 	handleImproveCortexStatus,
 	handleDedupTierAttribution,
 	handleCosineFpReport,
+	handleRepoCorpus,
 } from "./routes.js";
 // VC6C repair lives in its own module (routes-vector-cortex-repair.ts) so the
 // heal route file stays well under the 400-line extension soft limit.
@@ -162,5 +163,6 @@ export function dispatchRoutes(
 	if (handleImproveCortexStatus(req, res, ctx)) return true;
 	if (handleDedupTierAttribution(req, res, ctx)) return true;
 	if (handleCosineFpReport(req, res, ctx)) return true;
+	if (handleRepoCorpus(req, res, ctx)) return true;
 	return false;
 }

package/extensions/dashboard-server/routes-rag-settings-vector-cortex.ts

@@ -318,5 +318,17 @@ export const VECTOR_CORTEX_SETTINGS: SettingGroup = {
 			"Synthetic FP harness + L2 cosine threshold calibration (COS-FP-A): drives the deterministic synthetic-corpus grid sweep (scripts/cosine-fp/bench.mjs) that measures L2 cosine false-positive rates and emits a recommended default, and serves the reader-only GET /api/cosine-fp-report. Synthetic-corpus-only — never reads real session/ledger bytes. OFF = bench inert, endpoint 404s, no report write, L2_COSINE plain MEGACOMPACT_L2_THRESHOLD (byte-identical predecessor).",
 			true,
 		),
+		boolDirect(
+			"MEGACOMPACT_COSINE_FP_REAL",
+			"COS-FP-R Real-Corpus Validation",
+			"Real-corpus L2 cosine FP validation (COS-FP-R): runs the real-corpus grid sweep (scripts/cosine-fp/real-bench.mjs) against donated, consent-approved sessions only (SECURITY_PRIVACY Lifecycle/Consent), scoring FP/FN with Wilson intervals + session-grouped bootstrap, and appends CI-backed per-content-type recommendations to the cosine-threshold report. Runs only when a valid consented corpus exists; absent corpus = no_corpus (normative pre-donation state, nothing written). OFF = script inert, nothing executes, no writes (byte-identical predecessor).",
+			true,
+		),
+		boolDirect(
+			"MEGACOMPACT_REPO_CORPUS",
+			"Cross-Repo Corpus (REPO-A)",
+			"Cross-repo corpus preparation (REPO-A): the governed, consent-gated reader-only path that builds a pseudonymous corpus manifest from per-repo events.log slices and serves GET /api/repo-corpus answering which pseudonymous repos/sessions are in the corpus, what cross-repo overlap exists, and whether consent is active for each — counts + IDs + status only, never payload content. Consent is append-only (consent.mjs, CLI/ops-only); revocation freezes the builder + route. OFF = builder refuses to run, reader route 404s, single-repo recall untouched (byte-identical predecessor).",
+			true,
+		),
 	],
 };

package/extensions/dashboard-server/routes-repo-corpus.ts

@@ -0,0 +1,286 @@
+/**
+ * dashboard-server/routes-repo-corpus.ts — REPO-A cross-repo corpus status route.
+ *
+ * GET /api/repo-corpus — reader-only aggregate answering "which pseudonymous
+ * repos/sessions are in the consented corpus, what cross-repo overlap exists,
+ * and is consent active for each". Reads the corpus manifest + consent-state the
+ * CLI corpus-builder (scripts/repo-corpus/build.mjs) produced into the resolved
+ * corpus dir, memoized by {mtime,size} key with a 5s TTL (mirrors
+ * routes-cosine-fp.ts). Serves counts + IDs + status only — never payload
+ * content (EVAL-REDACT-002).
+ *
+ * IMPORTANT (distribution): this route does NOT import the builder scripts
+ * (scripts/ is excluded from package.json `files`). It reads the manifest files
+ * the builder wrote; the corpus dir is resolved from MEGACOMPACT_REPO_CORPUS_DIR
+ * (operator/test seam) or a bundled asset dir. The contract types ship in
+ * api-contracts/repo-corpus.ts.
+ *
+ * Flag-off (MEGACOMPACT_REPO_CORPUS=0) returns 404 — byte-identical
+ * predecessor. Absent corpus → awaiting_data (never a fabricated zero row).
+ *
+ * Guardrails: PREVENT-PI-004 (local filesystem read only), PREVENT-001
+ * (JSON.parse guarded), PREVENT-011 (no `any`).
+ */
+
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { readFileSync, statSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { RouteContext } from "./routes-core.js";
+import { sendJson } from "./routes-vector-cortex-shared.js";
+import { REPO_CORPUS_ENABLED } from "../../src/config.js";
+import { deriveVcStatus } from "./vc-status.js";
+import type {
+  RepoCorpusManifestV1,
+  RepoCorpusStatusV1,
+  RepoCorpusConsentStateV1,
+  RepoCorpusPerRepoStatusV1,
+} from "./api-contracts/repo-corpus.js";
+
+const MEMO_TTL_MS = 5000; // ≤5s per file state
+const MANIFEST_FILENAME = "manifest.json";
+const CONSENT_STATE_FILENAME = "consent-state.json";
+
+interface MemoEntry {
+  key: string;
+  at: number;
+  body: RepoCorpusStatusV1;
+}
+
+let memo: MemoEntry | null = null;
+
+/**
+ * Resolve the directory containing the corpus manifest + consent-state.
+ * 1. MEGACOMPACT_REPO_CORPUS_DIR (operator/test seam) — the canonical runtime
+ *    location where an operator ran the builder (or a test points a temp dir).
+ * 2. Bundled asset extension/dashboard-server/assets/ — the shipped location,
+ *    so a pre-built corpus can be shipped in the tarball (PREVENT-DIST-001);
+ *    tsc does not copy .json into dist, so probe co-located + repo-root.
+ * 3. Git-checkout well-known dir for dev use (scripts/repo-corpus/corpus-run/).
+ * Returns null when unresolvable — caller serves awaiting_data.
+ */
+export function repoCorpusDir(): string | null {
+  const override = process.env.MEGACOMPACT_REPO_CORPUS_DIR;
+  if (override) return override;
+  const here = dirname(fileURLToPath(import.meta.url));
+  const bundledCandidates = [
+    join(here, "assets"),
+    join(here, "..", "..", "..", "extensions", "dashboard-server", "assets"),
+  ];
+  for (const bundled of bundledCandidates) {
+    try {
+      // guardrails-allow PREVENT-PI-004: local bundled manifest stat (loopback)
+      statSync(join(bundled, MANIFEST_FILENAME));
+      return bundled;
+    } catch {
+      /* try next candidate, then fall through to git-checkout walk */
+    }
+  }
+  // Git-checkout corpus dir — dev path only.
+  let dir = here;
+  const rel = join("scripts", "repo-corpus", "corpus-run");
+  for (let i = 0; i < 8; i++) {
+    const candidate = join(dir, rel);
+    try {
+      // guardrails-allow PREVENT-PI-004: local corpus-run dir stat (loopback)
+      statSync(candidate);
+      return candidate;
+    } catch {
+      /* keep walking */
+    }
+    const next = dirname(dir);
+    if (next === dir) break;
+    dir = next;
+  }
+  return null;
+}
+
+/** Guarded JSON file read → parsed value or null (PREVENT-001, PREVENT-011). */
+function readJsonFile(p: string): unknown {
+  let raw: string;
+  try {
+    raw = readFileSync(p, "utf8"); // guardrails-allow PREVENT-PI-004: local corpus file read (loopback)
+  } catch {
+    return null;
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    return null;
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null;
+  return parsed;
+}
+
+function isRecord(v: unknown): v is Record<string, unknown> {
+  return !!v && typeof v === "object" && !Array.isArray(v);
+}
+
+/** Project a raw consent-state file (structurally guarded, PREVENT-011). */
+function readConsentState(corpusDir: string): RepoCorpusConsentStateV1 | null {
+  const raw = readJsonFile(join(corpusDir, CONSENT_STATE_FILENAME));
+  if (!isRecord(raw)) return null;
+  const perRepo = Array.isArray(raw.perRepo) ? raw.perRepo : [];
+  const rows = perRepo
+    .filter(isRecord)
+    .map((r) => ({
+      repoPseudonym:
+        typeof r.repoPseudonym === "string" ? r.repoPseudonym : "",
+      consentedCrossRepo: r.consentedCrossRepo === true || r.consentedCrossRepo === "true",
+      revokedAt: typeof r.revokedAt === "string" ? r.revokedAt : undefined,
+    }))
+    .filter((r) => r.repoPseudonym.length > 0);
+  return { schema: "repo-corpus-consent-state-v1", perRepo: rows };
+}
+
+/** Project a raw manifest file (structurally guarded, PREVENT-011). */
+function readManifest(corpusDir: string): RepoCorpusManifestV1 | null {
+  const raw = readJsonFile(join(corpusDir, MANIFEST_FILENAME));
+  if (!isRecord(raw)) return null;
+  if (raw.schema !== "repo-corpus-manifest-v1") return null;
+  const repos = Array.isArray(raw.repos)
+    ? raw.repos
+        .filter(isRecord)
+        .map((r) => ({
+          repoPseudonym:
+            typeof r.repoPseudonym === "string" ? r.repoPseudonym : "",
+          sessions: typeof r.sessions === "number" ? r.sessions : 0,
+          sessionIds: Array.isArray(r.sessionIds)
+            ? r.sessionIds.filter((x): x is string => typeof x === "string")
+            : [],
+          events: typeof r.events === "number" ? r.events : 0,
+          digest: typeof r.digest === "string" ? r.digest : "",
+        }))
+        .filter((r) => r.repoPseudonym.length > 0)
+    : [];
+  const overlaps = Array.isArray(raw.overlaps)
+    ? raw.overlaps
+        .filter(isRecord)
+        .map((o) => ({
+          repoA: typeof o.repoA === "string" ? o.repoA : "",
+          repoB: typeof o.repoB === "string" ? o.repoB : "",
+          sharedSessions: typeof o.sharedSessions === "number" ? o.sharedSessions : 0,
+          sharedIds: Array.isArray(o.sharedIds)
+            ? o.sharedIds.filter((x): x is string => typeof x === "string")
+            : [],
+        }))
+        .filter((o) => o.repoA.length > 0 && o.repoB.length > 0)
+    : [];
+  return {
+    schema: "repo-corpus-manifest-v1",
+    ownerPseudonym:
+      typeof raw.ownerPseudonym === "string" ? raw.ownerPseudonym : "",
+    datasetVersion:
+      typeof raw.datasetVersion === "string" ? raw.datasetVersion : "",
+    effectiveSeq: typeof raw.effectiveSeq === "number" ? raw.effectiveSeq : 0,
+    totalEvents: typeof raw.totalEvents === "number" ? raw.totalEvents : 0,
+    repos,
+    overlaps,
+  };
+}
+
+/**
+ * Build the perRepo status rows. A repo whose consent cannot be confirmed
+ * (missing from consent-state, or missing stateDir at the builder) degrades to
+ * consentedCrossRepo:false — never a crash, never a silent inclusion.
+ */
+function buildPerRepo(
+  manifest: RepoCorpusManifestV1,
+  consent: RepoCorpusConsentStateV1 | null,
+): RepoCorpusPerRepoStatusV1[] {
+  const consentByRepo = new Map<string, RepoCorpusConsentStateV1["perRepo"][number]>();
+  if (consent) {
+    for (const row of consent.perRepo) consentByRepo.set(row.repoPseudonym, row);
+  }
+  return manifest.repos.map((r) => {
+    const c = consentByRepo.get(r.repoPseudonym);
+    return {
+      repoPseudonym: r.repoPseudonym,
+      sessions: r.sessions,
+      consentedCrossRepo: c ? c.consentedCrossRepo : false,
+      revokedAt: c && c.revokedAt ? c.revokedAt : undefined,
+    };
+  });
+}
+
+function buildBody(corpusDir: string, manifest: RepoCorpusManifestV1): RepoCorpusStatusV1 {
+  const consent = readConsentState(corpusDir);
+  const perRepo = buildPerRepo(manifest, consent);
+  const hasData = manifest.repos.length > 0;
+  const status = deriveVcStatus({ enabled: true, hasData, structuralOnly: false });
+  return {
+    schema: "repo-corpus-status-v1",
+    corpus: manifest,
+    perRepo,
+    totalEvents: manifest.totalEvents,
+    status,
+  };
+}
+
+function awaitingDataBody(): RepoCorpusStatusV1 {
+  const status = deriveVcStatus({
+    enabled: true,
+    hasData: false,
+    structuralOnly: false,
+  });
+  return {
+    schema: "repo-corpus-status-v1",
+    corpus: null,
+    perRepo: [],
+    totalEvents: 0,
+    status,
+  };
+}
+
+export function handleRepoCorpus(
+  req: IncomingMessage,
+  res: ServerResponse,
+  _ctx: RouteContext,
+): boolean {
+  const url = req.url ?? "";
+  if (url !== "/api/repo-corpus") return false;
+  if (req.method !== "GET") {
+    sendJson(res, 405, { error: "method_not_allowed" });
+    return true;
+  }
+  if (!REPO_CORPUS_ENABLED()) {
+    // Flag-off: 404 — byte-identical predecessor.
+    sendJson(res, 404, { error: "not_found" });
+    return true;
+  }
+
+  const corpusDir = repoCorpusDir();
+  if (corpusDir === null) {
+    sendJson(res, 200, awaitingDataBody());
+    return true;
+  }
+
+  // Memoized-facts pattern: serve a still-fresh entry keyed on {mtime,size}.
+  let size = 0;
+  let mtime = 0;
+  try {
+    const s = statSync(join(corpusDir, MANIFEST_FILENAME)); // guardrails-allow PREVENT-PI-004: local stat (loopback)
+    size = s.size;
+    mtime = s.mtimeMs;
+  } catch {
+    // no manifest on disk yet
+  }
+  if (size === 0 && mtime === 0) {
+    sendJson(res, 200, awaitingDataBody());
+    return true;
+  }
+
+  const key = `${mtime}:${size}`;
+  const now = Date.now();
+  if (memo !== null && memo.key === key && now - memo.at <= MEMO_TTL_MS) {
+    sendJson(res, 200, memo.body);
+    return true;
+  }
+
+  const manifest = readManifest(corpusDir);
+  const body = manifest === null ? awaitingDataBody() : buildBody(corpusDir, manifest);
+  memo = { key, at: now, body };
+  sendJson(res, 200, body);
+  return true;
+}

package/extensions/dashboard-server/routes.ts

@@ -68,3 +68,4 @@ export {
 } from "./routes-cortex-improve.js";
 export { handleDedupTierAttribution } from "./routes-dedup-attribution.js";
 export { handleCosineFpReport } from "./routes-cosine-fp.js";
+export { handleRepoCorpus } from "./routes-repo-corpus.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.20.55",
+  "version": "0.20.57",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/config/repo-corpus.ts

@@ -0,0 +1,23 @@
+/**
+ * config/repo-corpus.ts — REPO-A cross-repo corpus preparation sprint flag.
+ *
+ * Positive sprint flag sibling (mirrors config/vector-cortex-cosfp.ts), NOT under
+ * a vector-cortex- prefix per the REPO-A spec: the reader route + corpus-builder
+ * gate on this flag. Root src/config.ts re-exports it so every existing import
+ * keeps resolving unchanged. Pi-agnostic, dependency-free (PREVENT-PI-004 /
+ * PREVENT-011).
+ */
+
+import { sprintFlag } from "./vector-cortex-flag.js";
+
+/**
+ * REPO-A — cross-repo corpus preparation. Default ON.
+ * `MEGACOMPACT_REPO_CORPUS=0` disables and is byte-identical to the predecessor:
+ * the reader route GET /api/repo-corpus 404s (no derived artifact written), the
+ * corpus-builder refuses to run, and the existing single-repo recall path is
+ * untouched. This flag MUST also be a dashboard SETTINGS toggle (visible in the
+ * config UI, never in EXCLUDED_SETTINGS). The builder + route perform only local
+ * filesystem reads / local `git` introspection (PREVENT-PI-004).
+ */
+export const REPO_CORPUS_ENABLED = (): boolean =>
+  sprintFlag("MEGACOMPACT_REPO_CORPUS");

package/src/config/vector-cortex-cosfp-real.ts

@@ -0,0 +1,37 @@
+/**
+ * config/vector-cortex-cosfp-real.ts — COS-FP-R real-corpus FP validation flag.
+ *
+ * Extracted from vector-cortex.ts so that barrel stays under the 300-line soft
+ * limit (soft-as-hard gate), exactly as vector-cortex-cosfp.ts and the other
+ * per-sprint flag siblings were. Gating the real-corpus L2 cosine false-positive
+ * validation: the `scripts/cosine-fp/real-bench.mjs` execution harness that runs
+ * only when a consented donated corpus exists, the execution-time report rows,
+ * and the evidence record. A pure-local, consent-gated tool (SECURITY_PRIVACY
+ * §Lifecycle/§Consent + EVAL-REDACT-002) — it never reads real session/ledger
+ * bytes without an explicit per-session consent record.
+ *
+ * The split is mechanical: COSINE_FP_REAL_ENABLED is byte-identical in name,
+ * semantics, and default to the sibling flag convention (positive, default ON),
+ * and vector-cortex.ts re-exports it so every existing
+ * `from "./config/vector-cortex.js"` import keeps resolving unchanged.
+ *
+ * Pi-agnostic, dependency-free (PREVENT-PI-004 / PREVENT-011).
+ */
+
+import { sprintFlag } from "./vector-cortex-flag.js";
+
+/**
+ * COS-FP-R — real-corpus L2 cosine FP validation against donated sessions.
+ * Default ON. `MEGACOMPACT_COSINE_FP_REAL=0` disables and is byte-identical to
+ * the predecessor: `scripts/cosine-fp/real-bench.mjs` is inert (nothing
+ * executes, no writes), no real-corpus report rows or evidence are written, and
+ * no threshold is adopted. When ON (default), execution still only proceeds when
+ * a valid consented corpus exists under `scripts/cosine-fp/corpus/` — absent
+ * corpus is the normative `no_corpus` pre-donation state, not a failure. Corpus
+ * bytes are read only for sessions with an explicit, append-only, revocable
+ * consent record (SECURITY_PRIVACY §Lifecycle/§Consent). This flag MUST also be
+ * a dashboard SETTINGS toggle (visible in config UI, never in EXCLUDED_SETTINGS).
+ * The harness performs only local filesystem reads (PREVENT-PI-004).
+ */
+export const COSINE_FP_REAL_ENABLED = (): boolean =>
+  sprintFlag("MEGACOMPACT_COSINE_FP_REAL");

package/src/config/vector-cortex.ts

@@ -64,6 +64,11 @@ export { ENC_0F_ENABLED } from "./vector-cortex-enc0f.js";
 export { ENC_0G_ENABLED } from "./vector-cortex-enc0g.js";
 // COS-FP-A synthetic FP harness flag extracted to vector-cortex-cosfp.ts.
 export { COSINE_FP_BENCH_ENABLED } from "./vector-cortex-cosfp.js";
+// COS-FP-R real-corpus FP validation flag extracted to vector-cortex-cosfp-real.ts.
+export { COSINE_FP_REAL_ENABLED } from "./vector-cortex-cosfp-real.js";
+// REPO-A cross-repo corpus prep flag extracted to repo-corpus.ts (no vector-cortex
+// prefix per spec); re-exported so root src/config.ts barrel resolves unchanged.
+export { REPO_CORPUS_ENABLED } from "./repo-corpus.js";
 export { ENC_1A_ENABLED } from "./vector-cortex-enc1a.js";
 export {
   ENC_1B_ENABLED,

package/src/config.ts

@@ -203,6 +203,8 @@ export {
   ENC_2BUDGET_MAX_MIB,
   ENC_2BUDGET_DEFAULT_MIB,
   COSINE_FP_BENCH_ENABLED,
+  COSINE_FP_REAL_ENABLED,
+  REPO_CORPUS_ENABLED,
   BREAKER_WINDOW_MS,
   BREAKER_MIN_ATTEMPTS,
   BREAKER_PERF_FAILURES,