pgkiln 0.69.0 → 0.70.0

package/src/cli/replace.ts

@@ -84,7 +84,9 @@ export const KEPT = ['app_access', 'api_client', 'session', 'sso_pending', 'save
   // (125) who changed the application last in the builder (App Builder list: Updated by)
   'app_change',
   // (126) outgoing calls the network access lists refused (Administration → Network access; a log, 7 days)
-  'network_refusal'];
+  'network_refusal',
+  // (136) rows queued for their embeddings (the vector providers travel in meta.app)
+  'embedding_queue'];
 /** Children of pages (replaced with their page). */
 const PAGE_CHILDREN = ['region', 'item', 'button', 'dynamic_action', 'validation', 'process', 'computation', 'branch'];
 /** References into replaced tables from kept data, repointed below: "table.column". */

package/src/runtime/ai-agents.ts

@@ -7,6 +7,7 @@ import { checkArgs, toolSchema, type ParamDef, type ToolDef } from './assistant.
 import { isAuthorized, pageAllowed } from './authz.ts';
 import { bindValues, dbg, publicError, stripSemicolon, type PageContext } from './context.ts';
 import { txContext } from './routes.ts';
+import { retrieve } from './search.ts';
 
 // AI agents and AI tools (migration 121; APEX 26.1 / 26.2: Shared Components
 // → AI Agents). Both are lists in meta.app (ai_agents, ai_tools), exported
@@ -31,11 +32,18 @@ import { txContext } from './routes.ts';
 //   values of that page; pgkiln refuses a page the user may not open and
 //   builds the URL itself (item values with their checksum, bound to app,
 //   page and user). The link is shown under the answer; nothing opens it.
+// - retrieve (136): RAG. The model gives a text (QUERY); pgkiln runs the
+//   tool's search configuration (vector, hybrid, full-text or standard) as
+//   the application's role in a read-only transaction, so row level
+//   security decides which rows the model sees, and returns the best rows'
+//   title, text and score (max_rows, default 5).
 // Every tool call is logged with its arguments (meta.ai_tool_call, the
 // Activity → AI usage page) and in the page's debug messages.
 
-export type AiToolType = 'sql' | 'function' | 'rest' | 'javascript' | 'navigate';
-export const AI_TOOL_TYPES: AiToolType[] = ['sql', 'function', 'rest', 'javascript', 'navigate'];
+export type AiToolType = 'sql' | 'function' | 'rest' | 'javascript' | 'navigate' | 'retrieve';
+export const AI_TOOL_TYPES: AiToolType[] = ['sql', 'function', 'rest', 'javascript', 'navigate', 'retrieve'];
+/** (136) Rows a Retrieve tool returns by default. */
+const RETRIEVE_ROWS = 5;
 export const EFFORTS = ['low', 'medium', 'high'] as const;
 
 export interface AiTool {
@@ -47,6 +55,8 @@ export interface AiTool {
   source?: string;
   action?: string;
   pages?: number[];
+  /** (136) retrieve: the search configuration (static ID) */
+  search_config?: string;
   parameters?: Record<string, ParamDef>;
   max_rows?: number;
   timeout_s?: number;
@@ -116,6 +126,9 @@ export function aiToolProblems(t: unknown): string[] {
     if (!Array.isArray(t.pages) || !t.pages.length || t.pages.length > 50 || t.pages.some((p: unknown) => !intIn(p, 1, 2_147_483_647)))
       out.push(`${what}: "pages" lists the page numbers the agent may offer (1 to 50 pages).`);
     if (t.parameters !== undefined) out.push(`${what}: a navigate tool has no parameters (the model picks a page, a link text and item values).`);
+  } else if (t.type === 'retrieve') {
+    if (typeof t.search_config !== 'string' || !/^[A-Z][A-Z0-9_]{0,59}$/.test(t.search_config)) out.push(`${what}: "search_config" names a search configuration (e.g. POLICIES).`);
+    if (t.parameters !== undefined) out.push(`${what}: a retrieve tool has no parameters (the model gives the text to look for).`);
   } else out.push(...paramProblems(what, t.parameters));
   if (t.max_rows !== undefined && !intIn(t.max_rows, 1, 500)) out.push(`${what}: maximum rows 1 to 500.`);
   if (t.timeout_s !== undefined && !intIn(t.timeout_s, 1, 60)) out.push(`${what}: time limit 1 to 60 seconds.`);
@@ -201,6 +214,14 @@ export interface PreparedTool {
 
 const asToolDef = (t: AiTool): ToolDef => ({ name: t.name, description: t.description, parameters: t.parameters, max_rows: t.max_rows });
 
+/** (136) The one parameter of a Retrieve tool. */
+const RETRIEVE_SCHEMA = {
+  type: 'object',
+  properties: { QUERY: { type: 'string', description: 'What to look for, in your own words (a question or a few words); the search finds rows by meaning.' } },
+  required: ['QUERY'],
+  additionalProperties: false,
+};
+
 /**
  * The agent's tools this user may use (their authorization schemes), with their schemas. Navigate tools keep
  * only the pages the user may open (checked here, in the caller's transaction). Needs ctx.client.
@@ -209,6 +230,10 @@ export async function prepareTools(ctx: PageContext, tools: AiTool[]): Promise<P
   const out: PreparedTool[] = [];
   for (const tool of tools) {
     if (!(await isAuthorized(ctx, tool.authz))) continue;
+    if (tool.type === 'retrieve') {
+      out.push({ tool, chat: { name: tool.name, description: `${tool.description} Returns the best matching rows (title, text, score) the user may read.`, parameters: RETRIEVE_SCHEMA } });
+      continue;
+    }
     if (tool.type !== 'navigate') {
       const extra = tool.type === 'javascript' ? ' It runs in the user\'s browser after your answer; it returns no data.' : '';
       out.push({ tool, chat: { name: tool.name, description: tool.description + extra, parameters: toolSchema(asToolDef(tool)) } });
@@ -309,6 +334,7 @@ export async function runAgentTool(ctx: PageContext, p: PreparedTool, input: Rec
   const { tool } = p;
   if (tool.type === 'navigate') return navigate(ctx, p, input, extras);
   if (tool.type === 'rest') return rest({ ...asToolDef(tool), type: 'rest', source: tool.source }, input);
+  if (tool.type === 'retrieve') return retrieveTool(ctx, tool, input);
   const checked = checkArgs(asToolDef(tool), input);
   if ('error' in checked) return { content: checked.error, isError: true };
   if (tool.type === 'javascript') {
@@ -336,6 +362,31 @@ export async function runAgentTool(ctx: PageContext, p: PreparedTool, input: Rec
   }
 }
 
+/**
+ * (136) A Retrieve tool: the search configuration as the application's role in a read-only transaction with the
+ * tool's time limit (row level security applies), its best rows as JSON for the model.
+ */
+async function retrieveTool(ctx: PageContext, tool: AiTool, input: Record<string, unknown>): Promise<AgentOutcome> {
+  for (const k of Object.keys(input)) if (k !== 'QUERY') return { content: `Unknown argument "${k}".`, isError: true };
+  const query = typeof input.QUERY === 'string' ? input.QUERY.replace(/\s+/g, ' ').trim().slice(0, 1000) : '';
+  if (!query) return { content: 'QUERY is the text to look for.', isError: true };
+  const max = tool.max_rows ?? RETRIEVE_ROWS;
+  const before = ctx.client;
+  try {
+    const found = await appTx(txContext(ctx), async (c) => {
+      await c.query(`select set_config('statement_timeout', $1, true)`, [`${tool.timeout_s ?? DEFAULT_TIMEOUT_S}s`]);
+      await c.query('set transaction read only');
+      ctx.client = c;
+      return retrieve(ctx, tool.search_config!, query, max);
+    });
+    return { content: JSON.stringify(found) };
+  } catch (e) {
+    return { content: await publicError(ctx, e, `AI tool ${tool.name}`), isError: true };
+  } finally {
+    ctx.client = before;
+  }
+}
+
 // ---------------------------------------------------------------- the log
 
 const MAX_ARG_CHARS = 4000;

package/src/runtime/search.ts

@@ -18,7 +18,14 @@
 // (between private-use markers that only then become <mark>, everything
 // else escaped); the text search configuration is checked against
 // pg_ts_config and passed as a parameter. With pg_trgm installed, a
-// configuration may suggest the closest words when nothing is found. The
+// configuration may suggest the closest words when nothing is found. Search
+// type vector (136): the search text embedded by the configuration's vector
+// provider (src/ai/embeddings.ts), the rows nearest by the vector column
+// (pgvector's cosine, L2 or inner product distance, operators
+// schema-qualified, the vector a query parameter), at least the minimum
+// similarity. Search type hybrid (136): the full-text and the vector
+// rankings merged by reciprocal rank fusion (1 / (60 + rank), summed per
+// key). Without pgvector both say what is missing. The
 // queries run as the application's role in the page's transaction, each in a
 // savepoint, so one configuration that fails shows its error and the others
 // still answer. Without JavaScript the field is a GET form; app.js searches
@@ -29,7 +36,9 @@ import { runtime, savepoint } from '../db.ts';
 import { html, raw, type Raw } from '../html.ts';
 import { icon } from '../icons.ts';
 import type { Region } from '../metadata.ts';
-import { bindValues, publicError, stripSemicolon, type PageContext } from './context.ts';
+import { embedQuery, vectorLiteral, vectorProviderFor } from '../ai/embeddings.ts';
+import { DISTANCES, distanceSql, isDistance, PGVECTOR_MISSING, vectorSchema, type Distance } from '../vector.ts';
+import { bindValues, dbg, publicError, stripSemicolon, type PageContext } from './context.ts';
 import { fillItems, linkAttrs } from './links.ts';
 import { listTree, type ListNode } from './lists.ts';
 
@@ -49,14 +58,24 @@ export interface SearchConfig {
   icon: string | null;
   link: { page?: number; items?: Record<string, string> } | null;
   max_results: number;
-  /** (135) standard: ILIKE per word · fulltext: PostgreSQL full-text search (later: vector, hybrid) */
-  search_type?: 'standard' | 'fulltext';
+  /** (135) standard: ILIKE per word · fulltext: PostgreSQL full-text search · (136) vector, hybrid */
+  search_type?: (typeof SEARCH_TYPES)[number];
   /** (135) fulltext: a tsvector column or expression; empty: to_tsvector over the searchable columns */
   fts_vector?: string | null;
   /** (135) fulltext: the text search configuration; empty: the application language's */
   fts_config?: string | null;
   /** (135) with pg_trgm: suggest the closest words when nothing is found */
   did_you_mean?: boolean;
+  /** (136) vector, hybrid: the vector provider (static ID) that embeds the search text */
+  vector_provider?: string | null;
+  /** (136) vector, hybrid: the source's vector column */
+  vector_column?: string | null;
+  /** (136) hybrid: the key that matches the full-text and the vector results */
+  key_column?: string | null;
+  /** (136) vector, hybrid: cosine, l2 or inner_product */
+  distance?: Distance;
+  /** (136) vector, hybrid: at least this similarity */
+  min_similarity?: number | null;
 }
 
 export interface SearchRegionConfig {
@@ -76,8 +95,11 @@ const TABLE = /^([A-Za-z_][A-Za-z0-9_$]{0,62}\.)?[A-Za-z_][A-Za-z0-9_$]{0,62}$/;
 /** A client, a pool or pgkiln's Db: something that runs a query with parameters. */
 type Queryable = { query: (sql: string, params?: unknown[]) => Promise<{ rows: any[] }> };
 
-/** (135) The search types (later: vector, hybrid). */
-export const SEARCH_TYPES = ['standard', 'fulltext'] as const;
+/** (135) The search types; (136) vector and hybrid. */
+export const SEARCH_TYPES = ['standard', 'fulltext', 'vector', 'hybrid'] as const;
+/** (136) Rows each ranking of a hybrid search contributes, and reciprocal rank fusion's constant. */
+const HYBRID_DEPTH = 50;
+const RRF_K = 60;
 const TS_CONFIG = /^[a-z_][a-z0-9_]{0,62}$/;
 
 /** (135) The text search configuration of a language: pgkiln's languages that PostgreSQL has a stemmer for. */
@@ -156,6 +178,20 @@ export function searchConfigProblems(c: Partial<SearchConfig>): string[] {
     if (c.fts_config && !TS_CONFIG.test(c.fts_config)) out.push('The text search configuration is a name, e.g. english, dutch or simple.');
     if (c.fts_vector && /;|--|\/\*/.test(c.fts_vector)) out.push('The tsvector is one column or expression (no ; or comments).');
   }
+  if (c.search_type === 'vector' || c.search_type === 'hybrid') {
+    if (c.source_type === 'list') out.push('Vector search needs a table or SQL source.');
+    if (!c.vector_provider || !/^[A-Z][A-Z0-9_]{0,59}$/.test(c.vector_provider)) out.push('Choose the vector provider that embeds the search text.');
+    if (!c.vector_column || !COLUMN.test(c.vector_column)) out.push('Name the vector column of the source (e.g. embedding).');
+    if (c.distance !== undefined && c.distance !== null && !isDistance(c.distance)) out.push('The distance is cosine, l2 or inner_product.');
+    if (c.min_similarity !== undefined && c.min_similarity !== null && !(Number.isFinite(Number(c.min_similarity)) && Number(c.min_similarity) >= -1 && Number(c.min_similarity) <= 1))
+      out.push('The minimum similarity is a number from -1 to 1 (empty: none).');
+  }
+  if (c.search_type === 'hybrid') {
+    if (!c.key_column || !COLUMN.test(c.key_column)) out.push('A hybrid search needs the key column that matches full-text and vector results (e.g. id).');
+    if (c.fts_config && !TS_CONFIG.test(c.fts_config)) out.push('The text search configuration is a name, e.g. english, dutch or simple.');
+    if (c.fts_vector && /;|--|\/\*/.test(c.fts_vector)) out.push('The tsvector is one column or expression (no ; or comments).');
+  }
+  if (c.key_column && !COLUMN.test(c.key_column)) out.push(`"${c.key_column}" is not a column name.`);
   return out;
 }
 
@@ -167,7 +203,8 @@ async function configsOf(ctx: PageContext, r: Region): Promise<SearchConfig[]> {
     await runtime.query<SearchConfig>(
       `select name, label, seq, source_type, source, searchable_columns, title_column, subtitle_column, description_column,
               badge_column, icon, link, max_results,
-              search_type, fts_vector, fts_config, did_you_mean
+              search_type, fts_vector, fts_config, did_you_mean,
+              vector_provider, vector_column, key_column, distance, min_similarity
          from meta.search_config
         where app_id = $1 and meta.build_option_on(app_id, build_option) and (cardinality($2::text[]) = 0 or name = any($2))
         order by seq, name`,
@@ -185,6 +222,8 @@ interface Hit {
   href: Raw | null;
   /** (135) full-text search: title, subtitle and description as highlighted by ts_headline */
   headlines?: { title: Raw; subtitle: Raw | null; description: Raw | null };
+  /** (136) vector: the similarity; hybrid: the fused score */
+  score?: number;
 }
 
 const text = (v: unknown) => (v === null || v === undefined ? null : typeof v === 'object' ? JSON.stringify(v) : String(v));
@@ -225,7 +264,9 @@ async function searchRows(ctx: PageContext, c: SearchConfig, words: string[], qu
   const from = await sourceFrom(ctx, c);
   let rows: Record<string, unknown>[];
   const fulltext = c.search_type === 'fulltext';
-  if (fulltext) {
+  if (c.search_type === 'vector' || c.search_type === 'hybrid') {
+    rows = await vectorRows(ctx, c, from, query);
+  } else if (fulltext) {
     // $1 the text search configuration (a name from pg_ts_config), $2 the user's text, $3 the headline options,
     // $4 the marker characters (taken out of the values first, so only ts_headline's markers become markup)
     const cfg = await textSearchConfig(ctx.client!, c.fts_config, ctx.app.language);
@@ -261,10 +302,71 @@ async function searchRows(ctx: PageContext, c: SearchConfig, words: string[], qu
           ? linkAttrs(ctx, c.link.page!, fillItems(c.link.items, (name) => col(row, name) ?? undefined))
           : null,
       ...(fulltext ? { headlines: { title: h(row, 'title') ?? html`${col(row, c.title_column) ?? ''}`, subtitle: h(row, 'subtitle'), description: h(row, 'description') } } : {}),
+      ...(typeof row['pgkiln$score'] === 'number' ? { score: Math.round((row['pgkiln$score'] as number) * 10_000) / 10_000 } : {}),
     })),
   };
 }
 
+/**
+ * (136) A vector or hybrid configuration's rows (up to max_results + 1), best first, each with "pgkiln$score"
+ * (vector: the similarity; hybrid: the fused score). Runs on ctx.client: the application's role, through RLS.
+ */
+async function vectorRows(ctx: PageContext, c: SearchConfig, from: string, query: string): Promise<Record<string, unknown>[]> {
+  const client = ctx.client!;
+  const schema = await vectorSchema(client);
+  if (!schema) throw new Error(PGVECTOR_MISSING);
+  const provider = await vectorProviderFor(ctx.app.id, c.vector_provider!);
+  const vector = await embedQuery(provider, query, { appId: ctx.app.id, pageNo: ctx.page?.page_no ?? null, user: ctx.user, debug: (l, x) => dbg(ctx, l, 'search', x) }, client);
+  const distance: Distance = isDistance(c.distance) ? c.distance : 'cosine';
+  const vcol = `t.${ident(c.vector_column!)}`;
+  const d = distanceSql(schema, distance, vcol, `$1::text::${ident(schema)}.vector`);
+  const minSim = c.min_similarity === null || c.min_similarity === undefined ? null : Number(c.min_similarity);
+  const sim = DISTANCES[distance].similarity;
+  if (c.search_type === 'vector') {
+    // the nearest rows (an HNSW index serves "order by distance limit n"), then those similar enough
+    const sql = `select s.*, ${sim('s."pgkiln$distance"')}::float8 as "pgkiln$score"
+      from (select t.*, ${d} as "pgkiln$distance" from ${from} where ${vcol} is not null order by ${d} limit ${c.max_results + 1}) s
+      where $2::float8 is null or ${sim('s."pgkiln$distance"')} >= $2::float8
+      order by s."pgkiln$distance"`;
+    return (await client.query(sql, [vectorLiteral(vector), minSim])).rows;
+  }
+  // hybrid: each ranking's best HYBRID_DEPTH rows, fused by reciprocal rank (1 / (60 + rank)) per key
+  const cfg = await textSearchConfig(client, c.fts_config, ctx.app.language);
+  const vec = tsvectorSql(c, '$3::regconfig');
+  const key = ident(c.key_column!);
+  const sql = `with v as (select k, row_number() over (order by dist) as r
+                  from (select t.${key} as k, ${d} as dist from ${from} where ${vcol} is not null order by ${d} limit ${HYBRID_DEPTH}) x
+                 where $2::float8 is null or ${sim('dist')} >= $2::float8),
+       f as (select k, row_number() over (order by rank desc) as r
+               from (select t.${key} as k, ts_rank_cd(${vec}, q.query) as rank from ${from}, websearch_to_tsquery($3::regconfig, $4) q(query)
+                      where ${vec} @@ q.query order by 2 desc limit ${HYBRID_DEPTH}) y),
+       s as (select k, sum(1.0 / (${RRF_K} + r))::float8 as score from (select k, r from v union all select k, r from f) z group by k)
+    select t.*, s.score as "pgkiln$score" from ${from} join s on s.k = t.${key}
+     order by s.score desc${c.title_column ? `, t.${ident(c.title_column)}` : ''} limit ${c.max_results + 1}`;
+  return (await client.query(sql, [vectorLiteral(vector), minSim, cfg, query])).rows;
+}
+
+/**
+ * (136) The Retrieve tool of an AI agent: a table or SQL configuration's best rows for a text, as the application's
+ * role (ctx.client, in the caller's read-only transaction): title, subtitle, description and badge, and the score.
+ */
+export async function retrieve(ctx: PageContext, name: string, query: string, max: number) {
+  const c = (await runtime.query<SearchConfig>(
+    `select name, label, seq, source_type, source, searchable_columns, title_column, subtitle_column, description_column,
+            badge_column, icon, link, max_results, search_type, fts_vector, fts_config, did_you_mean,
+            vector_provider, vector_column, key_column, distance, min_similarity
+       from meta.search_config where app_id = $1 and name = $2 and meta.build_option_on(app_id, build_option)`, [ctx.app.id, name.toUpperCase()])).rows[0];
+  if (!c) throw new Error(`There is no search configuration ${name.toUpperCase()}.`);
+  if (c.source_type === 'list') throw new Error(`Search configuration ${c.name} searches a list: a Retrieve tool needs a table or SQL source.`);
+  const problems = searchConfigProblems(c);
+  if (problems.length) throw new Error(`Search configuration ${c.name}: ${problems.join(' ')}`);
+  const { hits, more } = await searchRows(ctx, { ...c, max_results: max }, searchWords(query), query.slice(0, MAX_TEXT).trim());
+  return {
+    rows: hits.map((h) => ({ title: h.title, ...(h.subtitle ? { subtitle: h.subtitle } : {}), ...(h.description ? { text: h.description } : {}), ...(h.badge ? { badge: h.badge } : {}), ...(h.score !== undefined ? { score: h.score } : {}) })),
+    ...(more ? { more_rows: true } : {}),
+  };
+}
+
 /**
  * (135) "Did you mean": with pg_trgm installed, each search word that is not a word of the configuration's
  * searched columns replaced by the most similar one (similarity 0.3 or more), from the first 10,000 rows of the
@@ -350,14 +452,16 @@ async function searchResults(ctx: PageContext, r: Region, query: string): Promis
       if (!hits.length) continue;
       body = html`<ul class="search-hits">${hits.map((h) => {
         // (135) full-text results: ts_headline's highlights (stemmed words: "running" marks "run"); else the words
-        const title = h.headlines?.title ?? marked(h.title, words);
-        const subtitle = h.subtitle ? (h.headlines?.subtitle ?? marked(h.subtitle, words)) : null;
-        const description = h.description ? (h.headlines?.description ?? marked(h.description, words)) : null;
+        // (136) results by meaning: no word is marked (they need not contain the words)
+        const by = c.search_type === 'vector' || c.search_type === 'hybrid' ? [] : words;
+        const title = h.headlines?.title ?? marked(h.title, by);
+        const subtitle = h.subtitle ? (h.headlines?.subtitle ?? marked(h.subtitle, by)) : null;
+        const description = h.description ? (h.headlines?.description ?? marked(h.description, by)) : null;
         const inner = html`${h.icon ? html`<span class="search-icon">${icon(h.icon)}</span>` : ''}
           <span class="search-text"><span class="search-title">${title}</span>${
             subtitle ? html`<span class="search-subtitle">${subtitle}</span>` : ''}${
             description ? html`<span class="search-description">${description}</span>` : ''}</span>
-          ${h.badge ? html`<span class="badge-pill">${marked(h.badge, words)}</span>` : ''}`;
+          ${h.badge ? html`<span class="badge-pill">${marked(h.badge, by)}</span>` : ''}`;
         return html`<li>${h.href ? html`<a class="search-hit" ${h.href}>${inner}</a>` : html`<div class="search-hit">${inner}</div>`}</li>`;
       })}</ul>${more ? html`<p class="muted search-more">${ctx.locale.t('region.first_rows', { n: String(c.max_results) })}</p>` : ''}`;
     } catch (e) {

package/src/server.ts

@@ -5,6 +5,7 @@ import { startWorkflowRunner } from './workflow.ts';
 import { startProcessJobRunner } from './process-jobs.ts';
 import { startPushListener } from './push.ts';
 import { startMailListener } from './mail/mail.ts';
+import { startEmbeddingRunner } from './vector.ts';
 
 const app = await buildApp();
 await app.listen({ port: Number(process.env.PORT ?? 3100), host: process.env.HOST ?? '127.0.0.1' });
@@ -14,6 +15,8 @@ await startWorkflowRunner();
 await startProcessJobRunner();
 await startPushListener();
 await startMailListener();
+// (sprint 136) rows queued for their embeddings (EMBEDDINGS=off on servers that should not embed)
+startEmbeddingRunner();
 // (sprint 133, SEC-6) say at start-up what could lock people out: a changed instance key or address, the clock
 try {
   const { instanceChecks } = await import('./instance-checks.ts');

package/src/vector.ts

@@ -0,0 +1,288 @@
+import pg from 'pg';
+import { embedTexts, loadVectorProviders, vectorLiteral, vectorProviderProblems, DEFAULT_BATCH, DEFAULT_PER_MINUTE, type VectorProvider } from './ai/embeddings.ts';
+import { isAiError } from './ai/types.ts';
+import { owner } from './db.ts';
+import { devDbForApp } from './workspace-db.ts';
+
+// (sprint 136, AP-1) pgvector, and keeping vector columns current.
+//
+// pgvector is an extension that may be missing: stock postgres images don't have it. pgvectorStatus() says what
+// there is (as the SQL Workshop does for pg_cron), vectorType() the schema-qualified type name to use; nothing in
+// meta needs the extension.
+//
+// The embedding queue (migration 136): a vector column kept by a queue has an AFTER INSERT OR UPDATE OF … trigger
+// (meta.embedding_enqueue) that puts the row's key in meta.embedding_queue; Backfill queues the rows without an
+// embedding. embeddingTick() takes queued rows of one table and provider (FOR UPDATE SKIP LOCKED, so several
+// servers share the queue), reads their source columns, embeds them in batches of the provider's batch_size
+// within its requests per minute (and the application's daily AI limits), and writes the vectors. It reads and
+// writes the table with the developer connection of the application's workspace (the owner, or an isolated
+// workspace's developer login), as the SQL Workshop that created the column does. A failed batch is retried later
+// (1, 2, 4 … minutes; at most 6 attempts), its message kept for the builder.
+
+type Queryable = { query: (sql: string, params?: unknown[]) => Promise<{ rows: any[] }> };
+
+export interface PgvectorStatus {
+  /** the extension is created in this database */
+  installed: boolean;
+  version: string | null;
+  schema: string | null;
+  /** the server has the package (pg_available_extensions) */
+  available: boolean;
+  availableVersion: string | null;
+  /** the connection may create it (a superuser; pgvector is not a trusted extension) */
+  canCreate: boolean;
+}
+
+export async function pgvectorStatus(db: Queryable): Promise<PgvectorStatus> {
+  const r = (await db.query(
+    `select e.extversion as version, n.nspname as schema, a.default_version as available_version,
+            (select rolsuper from pg_catalog.pg_roles where rolname = current_user) as superuser
+       from (select 1) x
+       left join pg_catalog.pg_extension e on e.extname = 'vector'
+       left join pg_catalog.pg_namespace n on n.oid = e.extnamespace
+       left join pg_catalog.pg_available_extensions a on a.name = 'vector'`,
+  )).rows[0];
+  return {
+    installed: !!r.version, version: r.version ?? null, schema: r.schema ?? null,
+    available: !!r.available_version, availableVersion: r.available_version ?? null, canCreate: !!r.superuser,
+  };
+}
+
+/** The schema of pgvector when it is installed (its type and operators are used schema-qualified), else null. */
+export async function vectorSchema(db: Queryable): Promise<string | null> {
+  return (await db.query(`select n.nspname as s from pg_catalog.pg_extension e join pg_catalog.pg_namespace n on n.oid = e.extnamespace where e.extname = 'vector'`)).rows[0]?.s ?? null;
+}
+
+/** What to say when pgvector is missing (builder and runtime messages). */
+export const PGVECTOR_MISSING = 'The pgvector extension is not installed in this database: an administrator installs the package on the database server (e.g. apt install postgresql-17-pgvector, or the pgvector/pgvector Docker image) and creates it (Workspace utilities → Vector providers, or create extension vector as a superuser).';
+
+/** The distances of pgvector: operator, HNSW operator class, and the similarity (higher is closer) as SQL of `d`. */
+export const DISTANCES = {
+  cosine: { op: '<=>', opclass: 'vector_cosine_ops', similarity: (d: string) => `(1 - ${d})` },
+  l2: { op: '<->', opclass: 'vector_l2_ops', similarity: (d: string) => `(1 / (1 + ${d}))` },
+  inner_product: { op: '<#>', opclass: 'vector_ip_ops', similarity: (d: string) => `(-${d})` },
+} as const;
+export type Distance = keyof typeof DISTANCES;
+export const isDistance = (d: unknown): d is Distance => typeof d === 'string' && Object.hasOwn(DISTANCES, d);
+
+/** `a <=> b` with pgvector's operator schema-qualified (an application role's search_path may not have it). */
+export const distanceSql = (schema: string, d: Distance, a: string, b: string) => `${a} OPERATOR(${pg.escapeIdentifier(schema)}.${DISTANCES[d].op}) ${b}`;
+
+/** The dimensions of a vector column (its typmod), or null when it is no vector(n) column. */
+export async function vectorColumnDims(db: Queryable, table: string, column: string): Promise<number | null | undefined> {
+  const r = (await db.query(
+    `select t.typname = 'vector' as is_vector, a.atttypmod as dims
+       from pg_catalog.pg_attribute a join pg_catalog.pg_type t on t.oid = a.atttypid
+      where a.attrelid = $1::regclass and a.attname = $2 and a.attnum > 0 and not a.attisdropped`, [table, column])).rows[0];
+  if (!r) return undefined;
+  return r.is_vector && r.dims > 0 ? r.dims : null;
+}
+
+// ---------------------------------------------------------------- the queue
+
+const COLUMN = /^[a-z_][a-z0-9_$]{0,62}$/;
+const MAX_ATTEMPTS = 6;
+/** Requests per provider in the last minute (this process). */
+const sent = new Map<string, number[]>();
+
+function mayRequest(key: string, perMinute: number) {
+  const now = Date.now();
+  const times = (sent.get(key) ?? []).filter((t) => now - t < 60_000);
+  sent.set(key, times);
+  if (times.length >= perMinute) return false;
+  times.push(now);
+  return true;
+}
+
+interface Group { app_id: number; provider: string; tbl: number; vector_column: string; key_column: string; source_columns: string[] }
+
+/** Embed one batch of a group; returns the number of rows written (0: nothing to do or wait). */
+async function runBatch(g: Group, p: VectorProvider): Promise<number> {
+  const batch = p.type === 'ai_service' ? (p.batch_size ?? DEFAULT_BATCH) : 100;
+  if (p.type === 'ai_service' && !mayRequest(`${g.app_id}:${p.name}`, p.per_minute ?? DEFAULT_PER_MINUTE)) return 0;
+  return owner.tx(async (q) => {
+    const rows = (await q.query<{ id: string; key_value: string; queued_at: string }>(
+      `select id, key_value, queued_at::text from meta.embedding_queue
+        where app_id = $1 and provider = $2 and tbl = $3 and vector_column = $4 and key_column = $5 and source_columns = $6 and next_at <= now()
+        order by id limit $7 for update skip locked`,
+      [g.app_id, g.provider, g.tbl, g.vector_column, g.key_column, g.source_columns, batch])).rows;
+    if (!rows.length) return 0;
+    const ids = rows.map((r) => r.id);
+    const fail = async (message: string) => {
+      await q.query(
+        `update meta.embedding_queue set attempts = attempts + 1, last_error = left($2, 500),
+                next_at = now() + make_interval(mins => power(2, least(attempts, 6))::int)
+          where id = any($1::bigint[])`, [ids, message]);
+      return 0;
+    };
+    const db = await devDbForApp(g.app_id);
+    const table = (await db.query<{ t: string | null }>('select $1::oid::regclass::text as t from pg_catalog.pg_class where oid = $1', [g.tbl])).rows[0]?.t;
+    if (!table) {
+      // the table is gone
+      await q.query('delete from meta.embedding_queue where id = any($1::bigint[])', [ids]);
+      return 0;
+    }
+    if (![g.vector_column, g.key_column, ...g.source_columns].every((c) => COLUMN.test(c))) return fail('The trigger names a column that is not a plain name.');
+    const id = pg.escapeIdentifier;
+    const text = `meta.embedding_text(to_jsonb(t), $2::text[])`;
+    try {
+      let written = 0;
+      if (p.type === 'function') {
+        // a function provider: embedded in the database, as the developer connection (the trigger's own function)
+        await db.query(
+          `update ${table} t set ${id(g.vector_column)} = translate(meta.embed(${text}, $3, $4)::text, '{}', '[]')::text::${await vectorTypeOf(db)}
+            where t.${id(g.key_column)}::text = any($1::text[])`, [rows.map((r) => r.key_value), g.source_columns, p.name, g.app_id]);
+        written = rows.length;
+      } else {
+        const found = (await db.query<{ k: string; t: string | null }>(
+          `select t.${id(g.key_column)}::text as k, ${text} as t from ${table} t where t.${id(g.key_column)}::text = any($1::text[])`,
+          [rows.map((r) => r.key_value), g.source_columns])).rows;
+        const withText = found.filter((r) => r.t);
+        const vectors = withText.length ? await embedTexts(p, withText.map((r) => r.t!), { appId: g.app_id, user: null, pageNo: null }) : [];
+        const type = await vectorTypeOf(db);
+        // rows without text get no embedding; rows deleted meanwhile are simply gone
+        await db.query(
+          `update ${table} t set ${id(g.vector_column)} = x.v::${type}
+             from unnest($1::text[], $2::text[]) x(k, v) where t.${id(g.key_column)}::text = x.k`,
+          [[...withText.map((r) => r.k), ...found.filter((r) => !r.t).map((r) => r.k)], [...vectors.map(vectorLiteral), ...found.filter((r) => !r.t).map(() => null)]]);
+        written = found.length;
+      }
+      // done, unless the row was queued again meanwhile (the trigger moved queued_at)
+      await q.query(
+        `delete from meta.embedding_queue e using unnest($1::bigint[], $2::text[]) x(id, at) where e.id = x.id and e.queued_at::text = x.at`,
+        [ids, rows.map((r) => r.queued_at)]);
+      return Math.max(written, 1);
+    } catch (e) {
+      const message = isAiError(e) ? e.message : (e as Error).message;
+      return fail(message);
+    }
+  });
+}
+
+/** pgvector's type, schema-qualified, for casts. */
+async function vectorTypeOf(db: Queryable) {
+  const s = await vectorSchema(db);
+  if (!s) throw new Error(PGVECTOR_MISSING);
+  return `${pg.escapeIdentifier(s)}.vector`;
+}
+
+let lastPurge = 0;
+
+/** One round of the queue: a batch of each group with work (call again until it returns 0). */
+export async function embeddingTick(): Promise<number> {
+  if (Date.now() - lastPurge > 3_600_000) {
+    lastPurge = Date.now();
+    await owner.query(`delete from meta.embedding_queue where attempts >= $1 and next_at < now() - interval '7 days'`, [MAX_ATTEMPTS]);
+  }
+  const groups = (await owner.query<Group>(
+    `select distinct app_id, provider, tbl, vector_column, key_column, source_columns from meta.embedding_queue
+      where next_at <= now() and attempts < $1 limit 20`, [MAX_ATTEMPTS])).rows;
+  let done = 0;
+  const providers = new Map<number, VectorProvider[]>();
+  for (const g of groups) {
+    if (!providers.has(g.app_id)) providers.set(g.app_id, await loadVectorProviders(g.app_id));
+    const p = providers.get(g.app_id)!.find((x) => x.name === g.provider);
+    if (!p || vectorProviderProblems(p).length) {
+      await owner.query(
+        `update meta.embedding_queue set attempts = $5, last_error = $6, next_at = now() + interval '10 minutes'
+          where app_id = $1 and provider = $2 and tbl = $3 and vector_column = $4`,
+        [g.app_id, g.provider, g.tbl, g.vector_column, MAX_ATTEMPTS, `There is no valid vector provider ${g.provider} in application ${g.app_id}.`]);
+      continue;
+    }
+    try {
+      done += await runBatch(g, p);
+    } catch (e) {
+      console.error('embeddings:', (e as Error).message);
+    }
+  }
+  return done;
+}
+
+/** Work the queue until it is empty or waiting (tests, Backfill); at most `rounds` rounds. */
+export async function drainEmbeddings(rounds = 50) {
+  let total = 0;
+  for (let i = 0; i < rounds; i++) {
+    const n = await embeddingTick();
+    if (!n) break;
+    total += n;
+  }
+  return total;
+}
+
+let timer: NodeJS.Timeout | null = null;
+let busy = false;
+
+/** Kick the queue now (after a Backfill), without waiting for it (not with EMBEDDINGS=off). */
+export function kickEmbeddings() {
+  if (busy || process.env.EMBEDDINGS === 'off') return;
+  busy = true;
+  drainEmbeddings(20).catch((e) => console.error('embeddings:', (e as Error).message)).finally(() => (busy = false));
+}
+
+/** Started by server.ts; EMBEDDINGS=off on servers that should not embed, EMBEDDING_INTERVAL_S (default 5). */
+export function startEmbeddingRunner() {
+  if (process.env.EMBEDDINGS === 'off' || timer) return;
+  const every = Math.max(1, Number(process.env.EMBEDDING_INTERVAL_S ?? 5)) * 1000;
+  timer = setInterval(() => kickEmbeddings(), every);
+  timer.unref();
+}
+
+/** The queue of a vector column: waiting rows, failing rows and the last message. */
+export async function queueState(tbl: number, column: string) {
+  return (await owner.one<{ queued: number; failing: number; message: string | null }>(
+    `select count(*)::int as queued, count(*) filter (where attempts > 0)::int as failing,
+            (select last_error from meta.embedding_queue where tbl = $1 and vector_column = $2 and last_error is not null order by id desc limit 1) as message
+       from meta.embedding_queue where tbl = $1 and vector_column = $2`, [tbl, column]))!;
+}
+
+// ---------------------------------------------------------------- the triggers that keep vector columns
+
+export interface VectorTrigger {
+  table: string;
+  tbl: number;
+  trigger: string;
+  /** set: a function provider's embedding at once (BEFORE); queue: queued for the server (AFTER) */
+  mode: 'set' | 'queue';
+  appId: number;
+  provider: string;
+  vectorColumn: string;
+  keyColumn: string;
+  sourceColumns: string[];
+}
+
+/** The meta.embedding_set / meta.embedding_enqueue triggers (of one table, or of every table), with their arguments. */
+export async function vectorTriggers(db: Queryable, table?: string): Promise<VectorTrigger[]> {
+  const rows = (await db.query(
+    `select t.tgrelid::regclass::text as tbl_name, t.tgrelid::int as tbl, t.tgname as trigger, p.proname as fn, pg_catalog.pg_get_triggerdef(t.oid) as def
+       from pg_catalog.pg_trigger t join pg_catalog.pg_proc p on p.oid = t.tgfoid join pg_catalog.pg_namespace n on n.oid = p.pronamespace
+      where n.nspname = 'meta' and p.proname in ('embedding_set', 'embedding_enqueue') and not t.tgisinternal
+        and ($1::text is null or t.tgrelid = $1::regclass)
+      order by 1, 3`, [table ?? null])).rows;
+  const out: VectorTrigger[] = [];
+  for (const r of rows) {
+    const args = [...String(/EXECUTE FUNCTION [^(]+\((.*)\)\s*$/i.exec(r.def)?.[1] ?? '').matchAll(/'((?:[^']|'')*)'/g)].map((m) => m[1].replace(/''/g, "'"));
+    if (args.length < 5 || !/^\d+$/.test(args[0])) continue;
+    out.push({
+      table: r.tbl_name, tbl: r.tbl, trigger: r.trigger, mode: r.fn === 'embedding_set' ? 'set' : 'queue', appId: Number(args[0]),
+      provider: args[1].toUpperCase(), vectorColumn: args[2], keyColumn: args[3], sourceColumns: args.slice(4),
+    });
+  }
+  return out;
+}
+
+/** Queue rows of a vector column for the server: those without an embedding, or all of them. Returns how many. */
+export async function backfill(db: Queryable, t: VectorTrigger, all: boolean, max = 100_000): Promise<number> {
+  const id = pg.escapeIdentifier;
+  const keys = (await db.query(
+    `select t.${id(t.keyColumn)}::text as k from ${t.table} t ${all ? '' : `where t.${id(t.vectorColumn)} is null`} order by 1 limit ${max}`)).rows.map((r: { k: string }) => r.k);
+  for (let i = 0; i < keys.length; i += 5000) {
+    await owner.query(
+      `insert into meta.embedding_queue (app_id, provider, tbl, vector_column, key_column, source_columns, key_value)
+       select $1, $2, $3, $4, $5, $6, k from unnest($7::text[]) k
+       on conflict (tbl, vector_column, key_value) do update
+          set app_id = excluded.app_id, provider = excluded.provider, key_column = excluded.key_column, source_columns = excluded.source_columns,
+              queued_at = now(), next_at = now(), attempts = 0, last_error = null`,
+      [t.appId, t.provider, t.tbl, t.vectorColumn, t.keyColumn, t.sourceColumns, keys.slice(i, i + 5000)]);
+  }
+  return keys.length;
+}