relaybrain 0.0.0-stage → 0.2.0

package/backend/lib/falkor/index.js

@@ -0,0 +1,146 @@
+const { readConfig, loadFileConfig, saveFileConfig, configPath, validUrl } = require("./config");
+const { FalkorClient } = require("./client");
+const schema = require("./schema");
+
+const state = {
+  cfg: null,
+  client: null,
+  deps: {},
+  projects: new Map(),
+  started: false,
+};
+
+function projectStats(projectId) {
+  if (!state.projects.has(projectId)) {
+    state.projects.set(projectId, {
+      schemaVersion: null,
+      needsRebuild: false,
+      lastIngestAt: null,
+      lastIngestCount: 0,
+      bufferDrops: 0,
+      droppedRows: 0,
+      pendingLinks: 0,
+      lastError: null,
+    });
+  }
+  return state.projects.get(projectId);
+}
+
+function enabled() {
+  return Boolean(state.started && state.cfg?.enabled);
+}
+
+async function ensureProject(projectId) {
+  const info = await schema.ensure(state.client, projectId);
+  const stats = projectStats(projectId);
+  stats.schemaVersion = info.schemaVersion;
+  stats.needsRebuild = info.needsRebuild;
+  return info;
+}
+
+/** After any write: refresh the UI and let the analytics job know there's new structure. */
+function notify(projectId, payload = {}) {
+  try {
+    // Not "graph": that event name already carries the dependency graph to the dashboard.
+    state.deps.sse?.emit?.("memory-graph", { workspaceId: projectId, at: Date.now(), ...payload });
+  } catch {
+    /* UI refresh is best-effort */
+  }
+  require("./bus").emit("graph-written", { projectId });
+}
+
+function maskUrl(url) {
+  return url ? url.replace(/\/\/[^@/]*@/, "//***@") : null;
+}
+
+function health() {
+  return {
+    enabled: Boolean(state.cfg?.enabled),
+    running: state.started,
+    url: maskUrl(state.cfg?.url),
+    source: state.cfg?.source || null,
+    configFile: configPath(),
+    storeText: state.cfg?.storeText ?? null,
+    breaker: state.client ? state.client.status() : null,
+    projects: Object.fromEntries(state.projects),
+  };
+}
+
+/** Starts the client and every module once; used at boot and by a runtime enable. */
+function startGraph() {
+  if (state.started) return;
+  state.client = new FalkorClient(state.cfg);
+  state.started = true;
+  for (const name of MODULES) {
+    try {
+      require(name).start(module.exports);
+    } catch (err) {
+      console.warn(`[relay-falkor] ${name} failed to start: ${err.message || err}`);
+    }
+  }
+  state.client
+    .connection()
+    .then(() => console.log(`[relay-falkor] connected ${maskUrl(state.cfg.url)}`))
+    .catch((err) => console.warn(`[relay-falkor] not reachable yet (${err.message || err}); relay continues without it`));
+}
+
+/**
+ * Turns the graph on in a running relay serve (`relay graph enable`, Mission Control's
+ * Enable button). Saves the URL so the next start remembers it.
+ */
+async function enableAtRuntime(url) {
+  const target = String(url || "").trim() || "falkor://127.0.0.1:6379";
+  if (!validUrl(target)) throw Object.assign(new Error("URL must look like falkor://host:port"), { status: 400 });
+  const saved = { ...(loadFileConfig() || {}), url: target };
+  delete saved.disabled;
+  saveFileConfig(saved);
+  if (state.started) return { ...health(), note: "already running; restart relay serve to switch databases" };
+  state.cfg = readConfig();
+  if (!state.cfg.enabled) throw Object.assign(new Error("RELAY_FALKOR_DISABLED is set in this process"), { status: 409 });
+  startGraph();
+  try {
+    await state.client.connection();
+  } catch (err) {
+    return { ...health(), warning: `saved, but FalkorDB is not reachable yet: ${err.message || err}` };
+  }
+  return health();
+}
+
+function registerRoutes(app) {
+  app.get("/api/graph/health", (_req, res) => res.json(health()));
+  app.post("/api/graph/enable", async (req, res) => {
+    // Writes this machine's config: never through the room tunnel.
+    if (require("../roomAuth").isRemoteRequest(req)) return res.status(403).json({ ok: false, reason: "local requests only" });
+    try {
+      res.json({ ok: true, ...(await enableAtRuntime(req.body?.url)) });
+    } catch (err) {
+      res.status(err.status || 500).json({ ok: false, reason: String(err.message || err) });
+    }
+  });
+  // Registered even when disabled here: a room guest without FalkorDB still reads the host's graph.
+  require("./routes").register(app, module.exports);
+}
+
+/**
+ * Wires the memory graph into relay serve. Never throws: a broken or absent
+ * FalkorDB must leave every existing relay behavior untouched.
+ */
+function init(app, deps = {}) {
+  try {
+    state.cfg = readConfig();
+    state.deps = deps;
+    registerRoutes(app);
+    if (!state.cfg.enabled) {
+      console.log("[relay-falkor] memory graph off (run `relay graph enable`, or set RELAY_FALKOR_URL)");
+      return health();
+    }
+    startGraph();
+  } catch (err) {
+    console.warn(`[relay-falkor] init failed: ${err.message || err}`);
+  }
+  return health();
+}
+
+const MODULES = ["./ingest", "./coord", "./central", "./analytics"];
+
+module.exports = { init, state, enabled, ensureProject, projectStats, health, notify, enableAtRuntime };

package/backend/lib/falkor/ingest.js

@@ -0,0 +1,233 @@
+const { bus } = require("./bus");
+const cursor = require("./cursor");
+const intents = require("./intents");
+const { projectHistory, projectPatches, projectPeer } = require("./project");
+const { writeProjection } = require("./write");
+
+const DEBOUNCE_MS = 2000;
+const pending = new Map();
+const running = new Map();
+let api = null;
+
+function log(projectId, msg) {
+  console.log(`[relay-falkor] ${projectId}: ${msg}`);
+}
+
+// The cursor says what we sent; the graph's Meta.lastIngestAt says what it kept. RDB
+// snapshots can lose the last few seconds, and a FalkorDB can be wiped or swapped,
+// so a cursor newer than the graph means re-send everything (MERGE keeps it idempotent).
+const SNAPSHOT_GRACE_MS = 5000;
+const reconciled = new Set();
+
+function cursorFor(projectId, info, opts) {
+  const cur = cursor.load(projectId);
+  if (opts.full) return { ...cur, seen: [] };
+  if (!reconciled.has(projectId)) {
+    reconciled.add(projectId);
+    if (cur.seen.length && cur.updatedAt > (info?.lastIngestAt || 0) + SNAPSHOT_GRACE_MS) {
+      log(projectId, "graph is behind the ingest cursor (lost snapshot or new database); re-sending history");
+      return { ...cur, seen: [] };
+    }
+  }
+  return cur;
+}
+
+/** Projects + writes everything in `history` the cursor hasn't confirmed yet. */
+async function ingestHistory(projectId, history, workspacePath, opts = {}) {
+  const { client, state } = api;
+  const stats = api.projectStats(projectId);
+  const info = await api.ensureProject(projectId);
+  const cur = cursorFor(projectId, info, opts);
+  const seen = cursor.seenSet(cur);
+  const projection = projectHistory(history, {
+    workspacePath,
+    isNew: (id) => !seen.has(id),
+    storeText: state.cfg.storeText,
+  });
+  for (const info of projection.intents.values()) intents.remember(info);
+  if (!projection.sessions.length && !projection.turns.length && !projection.edits.length) return { skipped: true };
+
+  const t = Date.now();
+  const result = await writeProjection(client, projectId, projection);
+  // Stamp the graph before the cursor, so the cursor can never look older than what the graph holds.
+  await client.write(projectId, "MERGE (m:Meta {key:'meta'}) SET m.lastIngestAt = $now RETURN 1", { now: Date.now() });
+  cursor.commit(projectId, cur, (history || []).map((e) => e.id).filter(Boolean));
+  stats.lastIngestAt = Date.now();
+  stats.lastIngestCount = result.turnsCreated + result.editsCreated;
+  stats.droppedRows += projection.dropped;
+  stats.lastError = null;
+  log(projectId, `+${result.turnsCreated} turns, +${result.editsCreated} edits, ${result.sessions} sessions (${Date.now() - t}ms)`);
+  api.notify?.(projectId, result);
+  return result;
+}
+
+function schedule(projectId, payload) {
+  try {
+    const projected = projectHistory(payload.history, {
+      workspacePath: payload.workspacePath,
+      isNew: () => false,
+      storeText: false,
+    });
+    for (const info of projected.intents.values()) intents.remember(info);
+  } catch {
+    /* best-effort immediate intent cache update */
+  }
+  const entry = pending.get(projectId) || {};
+  entry.payload = payload;
+  clearTimeout(entry.timer);
+  entry.timer = setTimeout(() => flush(projectId), DEBOUNCE_MS);
+  if (entry.timer.unref) entry.timer.unref();
+  pending.set(projectId, entry);
+}
+
+async function flush(projectId) {
+  const entry = pending.get(projectId);
+  if (!entry) return;
+  if (running.get(projectId)) {
+    schedule(projectId, entry.payload);
+    return;
+  }
+  pending.delete(projectId);
+  if (!api.client.available()) return;
+  running.set(projectId, true);
+  try {
+    const { history, workspacePath } = entry.payload;
+    await ingestHistory(projectId, history, workspacePath);
+  } catch (err) {
+    // Replayable: the cursor wasn't committed, so the next sync re-sends these events.
+    api.projectStats(projectId).lastError = String(err.message || err);
+    if (!err.breaker) log(projectId, `ingest failed: ${err.message || err}`);
+  } finally {
+    running.delete(projectId);
+  }
+}
+
+// ---- patches: not replayable from history, so failed batches go back into a bounded buffer
+const PATCH_BUFFER_MAX = 2000;
+const patchQueues = new Map();
+
+function queuePatch(projectId, patch) {
+  const q = patchQueues.get(projectId) || { items: [], timer: null };
+  q.items.push(patch);
+  if (q.items.length > PATCH_BUFFER_MAX) {
+    q.items.splice(0, q.items.length - PATCH_BUFFER_MAX);
+    api.projectStats(projectId).bufferDrops += 1;
+  }
+  if (!q.timer) {
+    q.timer = setTimeout(() => flushPatches(projectId), 1000);
+    if (q.timer.unref) q.timer.unref();
+  }
+  patchQueues.set(projectId, q);
+}
+
+async function flushPatches(projectId) {
+  const q = patchQueues.get(projectId);
+  if (!q) return;
+  q.timer = null;
+  const items = q.items.splice(0);
+  if (!items.length) return;
+  const retry = () => {
+    q.items.unshift(...items);
+    q.timer = setTimeout(() => flushPatches(projectId), api.state.cfg.breakerCooldownMs);
+    if (q.timer.unref) q.timer.unref();
+  };
+  if (!api.client.available()) return retry();
+  try {
+    await api.ensureProject(projectId);
+    const result = await writeProjection(api.client, projectId, projectPatches(items, { storeText: api.state.cfg.storeText }));
+    if (result.editsCreated) log(projectId, `+${result.editsCreated} patch edits`);
+    api.notify?.(projectId, result);
+  } catch (err) {
+    api.projectStats(projectId).lastError = String(err.message || err);
+    retry();
+  }
+}
+
+// ---- room peers: debounced per (project, peer); content-addressed keys make re-pushes no-ops
+const peerSeen = new Map();
+const PEER_SEEN_MAX = 20000;
+
+function seenFor(projectId) {
+  if (!peerSeen.has(projectId)) peerSeen.set(projectId, new Set());
+  return peerSeen.get(projectId);
+}
+
+async function ingestPeer(projectId, { snapshot, login, trustRowOwners }) {
+  await api.ensureProject(projectId);
+  const seen = seenFor(projectId);
+  const projection = projectPeer(snapshot, {
+    login,
+    trustRowOwners,
+    isNew: (id) => !seen.has(id),
+    storeText: api.state.cfg.storeText,
+  });
+  for (const info of projection.intents.values()) intents.remember(info);
+  if (!projection.turns.length && !projection.edits.length) return { skipped: true };
+  const result = await writeProjection(api.client, projectId, projection);
+  for (const t of projection.turns) seen.add(`peer_${t.turnKey}`);
+  for (const e of projection.edits) seen.add(String(e.editKey).replace(/^e_/, ""));
+  for (const e of projection.edits) seen.add(e.editKey);
+  if (seen.size > PEER_SEEN_MAX) peerSeen.set(projectId, new Set([...seen].slice(-PEER_SEEN_MAX / 2)));
+  log(projectId, `peer ${login}: +${result.turnsCreated} turns, +${result.editsCreated} edits`);
+  api.notify?.(projectId, { peer: login, ...result });
+  return result;
+}
+
+const peerTimers = new Map();
+function schedulePeer(projectId, payload) {
+  try {
+    const projected = projectPeer(payload.snapshot, {
+      login: payload.login,
+      trustRowOwners: payload.trustRowOwners,
+      isNew: () => false,
+      storeText: false,
+    });
+    for (const info of projected.intents.values()) intents.remember(info);
+  } catch {
+    /* best-effort immediate intent cache update */
+  }
+  const key = `${projectId}|${payload.login}`;
+  const entry = peerTimers.get(key) || {};
+  entry.payload = payload;
+  clearTimeout(entry.timer);
+  entry.timer = setTimeout(async () => {
+    peerTimers.delete(key);
+    if (!api.client.available()) return;
+    try {
+      await ingestPeer(projectId, entry.payload);
+    } catch (err) {
+      api.projectStats(projectId).lastError = String(err.message || err);
+    }
+  }, DEBOUNCE_MS);
+  if (entry.timer.unref) entry.timer.unref();
+  peerTimers.set(key, entry);
+}
+
+function start(falkor) {
+  api = { ...falkor, client: falkor.state.client, state: falkor.state };
+  bus.on("history", ({ project, workspacePath, memory }) => {
+    if (!project?.id || !Array.isArray(memory?.history)) return;
+    schedule(project.id, { history: memory.history, workspacePath: workspacePath || project.path });
+  });
+  bus.on("patch", ({ projectId, patch }) => {
+    if (projectId && patch) queuePatch(projectId, patch);
+  });
+  bus.on("peer", ({ projectId, key, login, snapshot }) => {
+    if (!projectId || !snapshot) return;
+    // A guest's "host" bucket aggregates several people, so its row owners are meaningful;
+    // a single peer's push is forced to that peer's authenticated login.
+    const trustRowOwners = key === "host";
+    schedulePeer(projectId, { snapshot, login: login || key, trustRowOwners });
+  });
+}
+
+function configure(falkor) {
+  api = { ...falkor, client: falkor.state.client, state: falkor.state };
+}
+
+/** Forget per-process state, as a restart would. */
+function _resetForTests() {
+  reconciled.clear();
+}
+
+module.exports = { start, configure, ingestHistory, ingestPeer, flush, flushPatches, _resetForTests };

package/backend/lib/falkor/intents.js

@@ -0,0 +1,32 @@
+// In-process "what is this session trying to do" cache. Read on the lock deny path,
+// so it must stay a plain Map lookup: no I/O, no database.
+const bySession = new Map();
+const byAgentLabel = new Map();
+const MAX = 2000;
+
+function remember(info) {
+  if (!info?.sessionKey || !info.intent) return;
+  const prev = bySession.get(info.sessionKey);
+  if (prev && (prev.intentAt || 0) > (info.intentAt || 0)) return;
+  bySession.delete(info.sessionKey);
+  bySession.set(info.sessionKey, info);
+  if (bySession.size > MAX) bySession.delete(bySession.keys().next().value);
+  const latest = byAgentLabel.get(info.label);
+  if (!latest || (latest.intentAt || 0) <= (info.intentAt || 0)) byAgentLabel.set(info.label, info);
+}
+
+function forSession(sessionKey) {
+  return bySession.get(sessionKey) || null;
+}
+
+/** Fallback when a lock's session is unknown: the agent's most recent intent, low confidence. */
+function latestForLabel(label) {
+  return byAgentLabel.get(label) || null;
+}
+
+function clear() {
+  bySession.clear();
+  byAgentLabel.clear();
+}
+
+module.exports = { remember, forSession, latestForLabel, clear };

package/backend/lib/falkor/keys.js

@@ -0,0 +1,102 @@
+const crypto = require("crypto");
+
+// Fallback ids relay invents when an agent reports no session (unifyEvent → "session", lockAgentIdFor → "local").
+const SESSION_SENTINELS = new Set(["", "session", "local", "null", "undefined"]);
+const CONTENTION_BUCKET_MS = 10_000;
+
+function hash(parts, len = 20) {
+  return crypto.createHash("sha1").update(parts.map((p) => String(p ?? "")).join("␟")).digest("hex").slice(0, len);
+}
+
+function loginKey(login) {
+  const v = String(login || "").trim().toLowerCase();
+  return v || "local";
+}
+
+function agentKey(login, label) {
+  return `${loginKey(login)}::${String(label || "agent").trim()}`;
+}
+
+function isRealSession(sessionId) {
+  return !SESSION_SENTINELS.has(String(sessionId ?? "").trim().toLowerCase());
+}
+
+function sessionKey(label, sessionId) {
+  if (!isRealSession(sessionId)) return null;
+  return `${String(label || "agent").trim()}::${String(sessionId).trim()}`;
+}
+
+function day(ts) {
+  const d = new Date(Number(ts) || 0);
+  return Number.isNaN(d.getTime()) ? "unknown" : d.toISOString().slice(0, 10);
+}
+
+/** Groups session-less activity per agent, person and day so it never merges across people. */
+function bucketSessionKey(label, login, ts) {
+  return `${String(label || "agent").trim()}::nosession::${loginKey(login)}::${day(ts)}`;
+}
+
+/** `label:host:session` (or `label:host:session@login` from a remote room member). */
+function parseLockAgentId(agentId) {
+  const parts = String(agentId || "").split(":");
+  const label = parts[0] || "";
+  const host = parts[1] || "";
+  const rawSession = parts.slice(2).join(":");
+  const atIdx = rawSession.lastIndexOf("@");
+  const session = atIdx > 0 ? rawSession.slice(0, atIdx) : rawSession;
+  return { label, host, session, sessionKnown: isRealSession(session) };
+}
+
+function fileKey(p) {
+  if (p == null) return null;
+  let s = String(p).trim().replace(/\\/g, "/").replace(/\/{2,}/g, "/");
+  if (!s || /^[a-zA-Z]:\//.test(s) || /^[a-zA-Z]:$/.test(s) || s.startsWith("/")) return null;
+  s = s.replace(/^(\.\/)+/, "").replace(/\/$/, "");
+  if (!s || s === "." || s === ".." || s.startsWith("../") || s.includes("/../")) return null;
+  return s.toLowerCase();
+}
+
+function displayPath(p) {
+  return String(p || "").trim().replace(/\\/g, "/").replace(/^(\.\/)+/, "");
+}
+
+/** Content-addressed so a local turn and the same turn clipped in a peer snapshot converge. */
+function turnKey({ login, label, sessionId, ts, role, kind, text }) {
+  return `t_${hash([loginKey(login), label, sessionId, ts, role, kind, String(text || "").slice(0, 200)])}`;
+}
+
+function contentionKey({ kind, fileKey: fk, wanter, holder, ts }) {
+  const bucket = Math.floor((Number(ts) || 0) / CONTENTION_BUCKET_MS);
+  return `c_${hash([kind, fk, wanter, holder, bucket])}`;
+}
+
+function overlapKey(fk, editA, editB) {
+  const [a, b] = [String(editA), String(editB)].sort();
+  return `o_${hash([fk, a, b])}`;
+}
+
+/**
+ * Parsers record whether each event's time was in the transcript (event.tsSynthetic).
+ * History stored before they did falls back to the old tell: `mtime + index*1000` is fractional.
+ */
+function tsSynthetic(ts, flag) {
+  if (typeof flag === "boolean") return flag;
+  return !Number.isInteger(Number(ts));
+}
+
+module.exports = {
+  hash,
+  loginKey,
+  agentKey,
+  isRealSession,
+  sessionKey,
+  bucketSessionKey,
+  parseLockAgentId,
+  fileKey,
+  displayPath,
+  turnKey,
+  contentionKey,
+  overlapKey,
+  tsSynthetic,
+  CONTENTION_BUCKET_MS,
+};

package/backend/lib/falkor/migrations.js

@@ -0,0 +1,56 @@
+// Versioned in-place graph migrations (#18): run at schema.ensure() and recorded on
+// Meta.migrations so schema/ingest version bumps preserve history older than memory.json's cap
+// instead of requiring a destructive drop-and-rebuild.
+
+const MIGRATIONS = [
+  {
+    id: "001_meta_versions_v6",
+    description: "Ensure Session.stub defaults and stamp Meta to schemaVersion=1, ingestVersion=6 in place",
+    async up(client, projectId, { schemaVersion, ingestVersion }) {
+      await client.write(
+        projectId,
+        `MATCH (s:Session) WHERE s.stub IS NULL SET s.stub = false RETURN count(s) AS n`
+      );
+      await client.write(
+        projectId,
+        `MATCH (m:Meta {key: 'meta'})
+         SET m.schemaVersion = $schema, m.ingestVersion = $ingest, m.migratedAt = $now
+         RETURN 1 AS ok`,
+        { schema: schemaVersion, ingest: ingestVersion, now: Date.now() }
+      );
+    },
+  },
+];
+
+async function runMigrations(client, projectId, { schemaVersion, ingestVersion }) {
+  const rows = await client.read(
+    projectId,
+    "MATCH (m:Meta {key: 'meta'}) RETURN coalesce(m.migrations, []) AS migrations, m.schemaVersion AS s, m.ingestVersion AS i"
+  );
+  const meta = rows[0] || {};
+  const applied = new Set(Array.isArray(meta.migrations) ? meta.migrations : []);
+  const ran = [];
+
+  for (const mig of MIGRATIONS) {
+    if (applied.has(mig.id)) continue;
+    await mig.up(client, projectId, { schemaVersion, ingestVersion });
+    applied.add(mig.id);
+    ran.push(mig.id);
+  }
+
+  // If the graph had an older schemaVersion or ingestVersion, migrate it forward in place
+  // so existing Turn/Edit/Session/Decision history is preserved without dropping the graph.
+  if (ran.length || meta.s !== schemaVersion || meta.i !== ingestVersion) {
+    await client.write(
+      projectId,
+      `MATCH (m:Meta {key: 'meta'})
+       SET m.migrations = $migrations, m.schemaVersion = $schema, m.ingestVersion = $ingest
+       RETURN 1 AS ok`,
+      { migrations: [...applied], schema: schemaVersion, ingest: ingestVersion }
+    );
+  }
+
+  return { applied: [...applied], ran };
+}
+
+module.exports = { MIGRATIONS, runMigrations };