wednesdayai 0.4.12-dev.716 → 0.4.12-dev.719

package/extensions/learning-core/src/bounded-store-reads.test-helpers.ts

@@ -0,0 +1,387 @@
+import crypto from "node:crypto";
+import type { DatabaseSync } from "node:sqlite";
+import { describe, expect, it } from "vitest";
+import { readRankingPool, RANKING_SCAN_MAX_ROWS } from "./consolidation-judge.js";
+import { withCandidateDefaults } from "./extraction-candidate.js";
+import { createLearningOverview } from "./gateway-methods.js";
+import { withLessonDefaults } from "./lesson-record.js";
+import type { PgClientLike, PgPoolLike } from "./lesson-store-postgres.js";
+import type { SqliteDatabaseLike } from "./lesson-store-sqlite.js";
+import {
+  LESSON_EXCERPT_MAX_BYTES,
+  LESSON_PAGE_MAX_BYTES,
+  type ExtractionCandidate,
+  type LessonStore,
+  type SkillProposal,
+} from "./types.js";
+
+// Spec "Bounded store reads, scans, and success evidence": verification asserts rows AND bytes
+// read, not just result length. Every SQL leg runs through a meter that counts the rows the
+// database handed back and the payload bytes among them, so a materialize-then-filter read (the
+// pre-change readPayloads path) fails here even when its result is correct.
+
+type Meter = { rows: number; payloadBytes: number; excerptBytes: number };
+
+function newMeter(): Meter {
+  return { rows: 0, payloadBytes: 0, excerptBytes: 0 };
+}
+
+// One wrapped handle per test (the stores memoise schema init on the handle), metering into
+// whichever meter the last make() installed.
+let activeMeter: Meter = newMeter();
+
+function countRows(rows: unknown[]): void {
+  const meter = activeMeter;
+  for (const row of rows) {
+    meter.rows += 1;
+    for (const [column, value] of Object.entries(row as Record<string, unknown>)) {
+      if (column === "payload_json") {
+        meter.payloadBytes += Buffer.byteLength(
+          typeof value === "string" ? value : JSON.stringify(value),
+        );
+      } else if (column === "excerpt" && value instanceof Uint8Array) {
+        meter.excerptBytes += value.length;
+      }
+    }
+  }
+}
+
+export function meteredSqlite(db: DatabaseSync): SqliteDatabaseLike {
+  return {
+    exec: (sql) => db.exec(sql),
+    prepare(sql) {
+      const statement = db.prepare(sql);
+      return {
+        run: (...params) => statement.run(...(params as never[])),
+        get: (...params) => {
+          const row = statement.get(...(params as never[]));
+          countRows(row === undefined ? [] : [row]);
+          return row;
+        },
+        all: (...params) => {
+          const rows = statement.all(...(params as never[]));
+          countRows(rows);
+          return rows;
+        },
+      };
+    },
+  };
+}
+
+export function meteredPool(pool: PgPoolLike): PgPoolLike {
+  const wrap = async (result: Promise<{ rows: Record<string, unknown>[] }>) => {
+    const settled = await result;
+    countRows(settled.rows);
+    return settled;
+  };
+  return {
+    query: (sql, values) => wrap(pool.query(sql, values)),
+    async connect(): Promise<PgClientLike> {
+      const client = await pool.connect();
+      return {
+        query: (sql, values) => wrap(client.query(sql, values)),
+        release: () => client.release(),
+      };
+    },
+  };
+}
+
+const RUN_TOKEN = crypto.randomUUID().replaceAll("-", "");
+export const TEST_SCHEMA = `plugin_learning_core_bounded_${RUN_TOKEN}`;
+export const NOW = "2026-06-01T00:00:00.000Z";
+export const PAST = "2026-05-01T00:00:00.000Z";
+const FUTURE = "2027-01-01T00:00:00.000Z";
+
+export type SqlLeg = {
+  label: string;
+  /** Fixture sizes; the live leg is smaller only to keep a shared server fast. */
+  scale: number;
+  make: (meter: Meter) => LessonStore;
+  /** A store scoped to one row's own scope, over the same metered handle `make` reads through. */
+  writer: (scope: { workspaceId: string; agentId: string }) => LessonStore;
+};
+
+export function metering<T>(meter: Meter, value: T): T {
+  activeMeter = meter;
+  return value;
+}
+
+function iso(ms: number): string {
+  return new Date(Date.parse(PAST) + ms).toISOString();
+}
+
+export function candidate(
+  index: number,
+  status: ExtractionCandidate["status"],
+): ExtractionCandidate {
+  return withCandidateDefaults(
+    {
+      sessionId: `session-${String(index).padStart(6, "0")}`,
+      agentId: `agent-${index % 3}`,
+      workspaceId: `ws-${index % 2}`,
+      excerpt: `candidate ${index} ${"x".repeat(900)}`,
+      tags: index % 2 === 0 ? ["correction"] : ["failure"],
+      status,
+    },
+    iso(index * 1000),
+  );
+}
+
+export function lesson(
+  index: number,
+  text: string,
+  opts?: { invalidAt?: string; validFrom?: string },
+) {
+  const at = iso(index * 1000);
+  return withLessonDefaults(
+    {
+      id: `lesson-${String(index).padStart(6, "0")}`,
+      scope: { agentId: `agent-${index % 3}`, workspaceId: `ws-${index % 2}` },
+      kind: "procedure" as const,
+      text,
+      confidence: 0.9,
+      occurrences: 1,
+      distinctSessionIds: [`s-${index}`],
+      evidence: [],
+      createdAt: at,
+      updatedAt: at,
+      ...(opts?.validFrom ? { validFrom: opts.validFrom } : {}),
+      ...(opts?.invalidAt ? { invalidAt: opts.invalidAt } : {}),
+    },
+    at,
+  );
+}
+
+export function proposal(index: number, status: SkillProposal["status"]): SkillProposal {
+  return {
+    id: `proposal-${index}`,
+    lessonIds: [`lesson-${String(index).padStart(6, "0")}`],
+    status,
+    skillName: `skill-${index}`,
+    title: `Proposal ${index}`,
+    body: "b".repeat(4000),
+    scannerFindings: [],
+    createdAt: iso(index),
+    updatedAt: iso(index),
+  };
+}
+
+// Mixed 1-, 2- and 3-byte characters, so a byte cut can land inside a character.
+const LONG_TEXT = `${"é".repeat(3000)}${"界".repeat(2000)} tail`;
+
+async function seedScoped(
+  writer: (scope: { workspaceId: string; agentId: string }) => LessonStore,
+  items: Array<{
+    scope: { workspaceId: string; agentId: string };
+    write: (store: LessonStore) => Promise<void>;
+  }>,
+): Promise<void> {
+  for (const item of items) {
+    await item.write(writer(item.scope));
+  }
+}
+
+export function registerBoundedSqlLegSuite(leg: SqlLeg): void {
+  describe(`bounded SQL store reads — ${leg.label}`, () => {
+    // Writes go through a store scoped to the row's own scope (as production capture does), reads
+    // through the unscoped store the gateway and the passes use.
+    const scopedWriter = (meter: Meter) => (scope: { workspaceId: string; agentId: string }) => {
+      activeMeter = meter;
+      return leg.writer(scope);
+    };
+    const scopeOf = (index: number) => ({
+      agentId: `agent-${index % 3}`,
+      workspaceId: `ws-${index % 2}`,
+    });
+
+    it("pending candidates: rows and payload bytes read are the pending set only, keyset-paged", async () => {
+      const total = Math.round(2000 * leg.scale);
+      const pendingEvery = 100;
+      const seedMeter = newMeter();
+      const writer = scopedWriter(seedMeter);
+      const all: ExtractionCandidate[] = [];
+      for (let index = 0; index < total; index += 1) {
+        const entry = candidate(index, index % pendingEvery === 0 ? "pending" : "processed");
+        all.push(entry);
+        await writer(scopeOf(index)).upsertCandidate(entry);
+      }
+      const expected = all.filter((entry) => entry.status === "pending");
+      const pendingBytes = expected.reduce(
+        (sum, entry) => sum + Buffer.byteLength(JSON.stringify(entry)),
+        0,
+      );
+      const allBytes = all.reduce(
+        (sum, entry) => sum + Buffer.byteLength(JSON.stringify(entry)),
+        0,
+      );
+
+      const meter = newMeter();
+      const store = leg.make(meter);
+      const pending = await store.listPendingCandidates();
+      expect(pending.map((entry) => entry.sessionId)).toEqual(
+        expected.map((entry) => entry.sessionId),
+      );
+      expect(meter.rows, "rows read must be the pending rows, not every candidate").toBe(
+        expected.length,
+      );
+      // jsonb re-serialization may reorder keys but never grows a row by more than its key quotes.
+      expect(meter.payloadBytes).toBeLessThanOrEqual(pendingBytes * 1.1);
+      expect(meter.payloadBytes).toBeLessThan(allBytes / 20);
+
+      // Keyset pages: LIMIT applies in SQL, pages chain without gaps or repeats, tags filter.
+      const pageMeter = newMeter();
+      const paged = leg.make(pageMeter);
+      const first = await paged.listPendingCandidatesPage({ limit: 7 });
+      expect(pageMeter.rows).toBe(7);
+      const seen = [...first.candidates];
+      let cursor = first.nextCursor;
+      while (cursor) {
+        const page = await paged.listPendingCandidatesPage({ after: cursor, limit: 7 });
+        seen.push(...page.candidates);
+        cursor = page.nextCursor;
+      }
+      expect(seen.map((entry) => entry.sessionId)).toEqual(
+        expected.map((entry) => entry.sessionId),
+      );
+      const tagged = await paged.listPendingCandidates({ tags: ["failure"] });
+      expect(tagged.map((entry) => entry.sessionId)).toEqual(
+        expected.filter((entry) => entry.tags.includes("failure")).map((entry) => entry.sessionId),
+      );
+      const limited = await paged.listPendingCandidates({ limit: 3 });
+      expect(limited.map((entry) => entry.sessionId)).toEqual(
+        expected.slice(0, 3).map((entry) => entry.sessionId),
+      );
+    });
+
+    it("similar-lesson ranking reads { id, excerpt } pages: no payload bytes, 2 KiB rows, 256 KiB pages", async () => {
+      const activeCount = Math.round(450 * leg.scale);
+      const seedMeter = newMeter();
+      const writer = scopedWriter(seedMeter);
+      const activeIds: string[] = [];
+      for (let index = 0; index < activeCount; index += 1) {
+        const entry = lesson(index, index % 5 === 0 ? `short ${index}` : `${LONG_TEXT} ${index}`);
+        activeIds.push(entry.id);
+        await writer(scopeOf(index)).upsertLesson(entry);
+      }
+      // Inactive rows the SQL predicate must drop: invalidated, and not yet valid.
+      for (let index = activeCount; index < activeCount + 40; index += 1) {
+        await writer(scopeOf(index)).upsertLesson(
+          lesson(index, LONG_TEXT, index % 2 === 0 ? { invalidAt: PAST } : { validFrom: FUTURE }),
+        );
+      }
+
+      const meter = newMeter();
+      const store = leg.make(meter);
+      const first = await store.listActiveLessonsPage({ now: NOW, limit: 200 });
+      expect(meter.payloadBytes, "no payload_json may leave the database").toBe(0);
+      expect(first.lessons.length).toBeLessThanOrEqual(200);
+      expect(first.nextCursor).toBeDefined();
+      const firstBytes = first.lessons.reduce(
+        (sum, entry) => sum + Buffer.byteLength(entry.excerpt),
+        0,
+      );
+      expect(firstBytes).toBeLessThanOrEqual(LESSON_PAGE_MAX_BYTES);
+      expect(
+        meter.excerptBytes,
+        "the database returned at most one page of excerpts",
+      ).toBeLessThanOrEqual(LESSON_PAGE_MAX_BYTES);
+      for (const entry of first.lessons) {
+        expect(Buffer.byteLength(entry.excerpt)).toBeLessThanOrEqual(LESSON_EXCERPT_MAX_BYTES);
+        expect(entry.excerpt).not.toContain("�");
+      }
+
+      const walkMeter = newMeter();
+      const walker = leg.make(walkMeter);
+      const pool = await readRankingPool(walker, NOW);
+      expect(pool.truncated).toBe(false);
+      expect(pool.lessons.map((entry) => entry.id).toSorted()).toEqual(activeIds.toSorted());
+      expect(walkMeter.rows, "every active row read exactly once, inactive never").toBe(
+        activeCount,
+      );
+      expect(walkMeter.payloadBytes).toBe(0);
+    });
+
+    it("the ranking scan stops at its 2,000-row ceiling", async () => {
+      if (leg.label.includes("live")) {
+        return;
+      }
+      const writer = scopedWriter(newMeter());
+      for (let index = 0; index < RANKING_SCAN_MAX_ROWS + 50; index += 1) {
+        await writer(scopeOf(index)).upsertLesson(lesson(index, `tiny ${index}`));
+      }
+      const meter = newMeter();
+      const pool = await readRankingPool(leg.make(meter), NOW);
+      expect(pool.truncated).toBe(true);
+      expect(pool.lessons).toHaveLength(RANKING_SCAN_MAX_ROWS);
+      expect(meter.rows).toBe(RANKING_SCAN_MAX_ROWS);
+    });
+
+    it("overview counts are SQL aggregates: a handful of rows, no payload bytes", async () => {
+      const seedMeter = newMeter();
+      const writer = scopedWriter(seedMeter);
+      const lessons = Math.round(300 * leg.scale);
+      const items: Parameters<typeof seedScoped>[1] = [];
+      for (let index = 0; index < lessons; index += 1) {
+        items.push({
+          scope: scopeOf(index),
+          write: (store) =>
+            store.upsertLesson(
+              lesson(index, "t", index % 10 === 0 ? { invalidAt: PAST } : undefined),
+            ),
+        });
+      }
+      const statuses: SkillProposal["status"][] = ["pending", "applied", "rejected", "quarantined"];
+      for (let index = 0; index < 40; index += 1) {
+        items.push({
+          scope: scopeOf(index),
+          write: (store) => store.upsertProposal(proposal(index, statuses[index % 4])),
+        });
+      }
+      await seedScoped(writer, items);
+
+      const meter = newMeter();
+      const store = leg.make(meter);
+      const counts = await store.countOverview({ now: NOW });
+      expect(counts).toEqual({
+        proposals: { pending: 10, applied: 10, rejected: 10, quarantined: 10 },
+        lessons: lessons - lessons / 10,
+        scopes: 6,
+        truncated: false,
+      });
+      expect(meter.payloadBytes).toBe(0);
+      expect(meter.rows).toBeLessThanOrEqual(8);
+
+      // The real entry point: learning.overview over the same store.
+      const calls: Array<{ ok: boolean; payload?: unknown }> = [];
+      const handlerMeter = newMeter();
+      await createLearningOverview({
+        readResolution: async () => ({}),
+        lessonStore: () => leg.make(handlerMeter),
+        now: () => NOW,
+      })({
+        params: {},
+        respond: (ok: boolean, payload?: unknown) => calls.push({ ok, payload }),
+      } as never);
+      expect(calls[0]).toMatchObject({
+        ok: true,
+        payload: { lessons: counts.lessons, scopes: 6, truncated: false },
+      });
+      expect(handlerMeter.payloadBytes).toBe(0);
+    });
+
+    it("getLesson and getLessonsByIds read only the named rows", async () => {
+      const writer = scopedWriter(newMeter());
+      for (let index = 0; index < 200; index += 1) {
+        await writer(scopeOf(index)).upsertLesson(lesson(index, LONG_TEXT));
+      }
+      const meter = newMeter();
+      const store = leg.make(meter);
+      const found = await store.getLessonsByIds(["lesson-000003", "lesson-000150", "missing"]);
+      expect(found.map((entry) => entry.id).toSorted()).toEqual(["lesson-000003", "lesson-000150"]);
+      expect(meter.rows).toBe(2);
+      const one = newMeter();
+      expect((await leg.make(one).getLesson("lesson-000007"))?.id).toBe("lesson-000007");
+      expect(one.rows).toBe(1);
+    });
+  });
+}

package/extensions/learning-core/src/lesson-store-conformance.pg.integration.test.ts

@@ -0,0 +1,77 @@
+import { afterAll, afterEach, beforeAll, describe, expect } from "vitest";
+import {
+  PG_TABLE,
+  registerLessonStoreConformance,
+  RUN_ID,
+  TEST_SCHEMA,
+  type Backend,
+} from "./lesson-store-conformance.test-helpers.js";
+import {
+  createDefaultLearningPgPool,
+  PostgresLessonStore,
+  type PgPoolLike,
+} from "./lesson-store-postgres.js";
+
+// Live-server leg of the LessonStore conformance suite (`pnpm test:pg`): the contract the fs-jsonl,
+// sqlite and in-memory-double legs run in lesson-store-conformance.test.ts, against a real
+// Postgres. This leg, not the double, is the authority on real dialect behaviour.
+const databaseUrl = process.env.OPENCLAW_DATABASE_URL;
+const live = process.env.OPENCLAW_LIVE_PG_TEST === "1" && Boolean(databaseUrl);
+
+type ClosablePool = PgPoolLike & { end?: () => Promise<void> };
+let sharedPool: ClosablePool;
+
+async function cleanupPostgresRows(): Promise<void> {
+  // The table only exists once the postgres leg has run its DDL; probing avoids swallowing a
+  // real error in a bare catch.
+  const probe = await sharedPool.query("SELECT to_regclass($1) AS reg", [PG_TABLE]);
+  if (!probe.rows[0]?.reg) {
+    return;
+  }
+  await sharedPool.query(`DELETE FROM ${PG_TABLE} WHERE workspace_id LIKE $1`, [`${RUN_ID}%`]);
+}
+
+const liveBackend: Backend = {
+  name: "postgres",
+  label: "postgres (live server)",
+  make: async (scope) => new PostgresLessonStore({ pool: sharedPool, schema: TEST_SCHEMA, scope }),
+  snapshot: async () => {
+    const result = await sharedPool.query(
+      `SELECT * FROM ${PG_TABLE} WHERE workspace_id LIKE $1
+       ORDER BY workspace_id, agent_id, record_type, record_id`,
+      [`${RUN_ID}%`],
+    );
+    return JSON.stringify(result.rows);
+  },
+  assertSearchIndex: async () => {
+    const cols = await sharedPool.query(
+      `SELECT column_name FROM information_schema.columns
+       WHERE table_schema = $1 AND table_name = 'learning_records' AND column_name = 'search_tsv'`,
+      [TEST_SCHEMA],
+    );
+    expect(cols.rows.length).toBeGreaterThan(0);
+    const idx = await sharedPool.query(
+      `SELECT indexname FROM pg_indexes
+       WHERE schemaname = $1 AND tablename = 'learning_records' AND indexdef ILIKE '%gin%'`,
+      [TEST_SCHEMA],
+    );
+    expect(idx.rows.length).toBeGreaterThan(0);
+  },
+};
+
+describe.skipIf(!live)("live postgres", () => {
+  beforeAll(() => {
+    sharedPool = createDefaultLearningPgPool(databaseUrl!);
+  });
+
+  afterEach(async () => {
+    await cleanupPostgresRows();
+  });
+
+  afterAll(async () => {
+    await sharedPool.query(`DROP SCHEMA IF EXISTS ${TEST_SCHEMA} CASCADE`);
+    await sharedPool.end?.();
+  });
+
+  registerLessonStoreConformance(liveBackend);
+});