pgkiln 0.68.0 → 0.70.0

package/src/ai/embeddings.ts

@@ -0,0 +1,197 @@
+import OpenAI from 'openai';
+import { owner } from '../db.ts';
+import { openAiError } from './openai.ts';
+import { baseUrlFor, checkDailyLimit, logUsage, serviceForApp, serviceKey, type CallInfo } from './service.ts';
+import { AiError } from './types.ts';
+
+// (sprint 136, AP-1; APEX: Vector Providers, APEX_AI.GET_VECTOR_EMBEDDINGS) Vector providers turn text into an
+// embedding. They are a JSON list in meta.app (vector_providers, migration 136), exported with the application:
+//
+//  - ai_service: an AI service of the installation (OpenAI, Gemini, or any OpenAI-compatible endpoint such as
+//    Ollama) and an embedding model (text-embedding-3-small, gemini-embedding-001, nomic-embed-text …), called with
+//    the official openai SDK's embeddings API. The key stays in meta.ai_service (encrypted, write-only, never
+//    exported); the application must be allowed to use the service and its daily limits apply; every call is
+//    logged in meta.ai_usage (source "embedding": tokens, model, duration, never the text).
+//  - function: a SQL or PL/pgSQL function (text → vector, real[] or float8[]), called by meta.embed() as the
+//    role that runs the query (security invoker), so its grants apply.
+//
+// The dimensions are part of the definition: every embedding is checked against them, and the builder checks
+// them against the vector column (vector(n)) a search configuration or a vector column helper uses.
+
+export type VectorProviderType = 'ai_service' | 'function';
+export const VECTOR_PROVIDER_TYPES: VectorProviderType[] = ['ai_service', 'function'];
+
+export interface VectorProvider {
+  name: string;
+  description?: string;
+  type: VectorProviderType;
+  /** ai_service: the AI service's name */
+  service?: string;
+  /** ai_service: the embedding model, exactly as the provider names it */
+  model?: string;
+  dimensions: number;
+  /** function: schema.name */
+  function?: string;
+  /** ai_service: texts per request (background embedding; default 64) */
+  batch_size?: number;
+  /** ai_service: requests per minute (background embedding; default 60) */
+  per_minute?: number;
+}
+
+export const PROVIDER_NAME = /^[A-Z][A-Z0-9_]{0,59}$/;
+const SERVICE_NAME = /^[A-Za-z][A-Za-z0-9_]{0,59}$/;
+const MODEL = /^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,199}$/;
+export const FUNCTION_NAME = /^[A-Za-z_][A-Za-z0-9_]{0,62}\.[A-Za-z_][A-Za-z0-9_]{0,62}$/;
+/** pgvector's vector type takes up to 16,000 dimensions (an HNSW index up to 2,000). */
+export const MAX_DIMENSIONS = 16_000;
+export const HNSW_MAX_DIMENSIONS = 2_000;
+export const DEFAULT_BATCH = 64;
+export const DEFAULT_PER_MINUTE = 60;
+/** Characters of one text sent to a provider (longer texts are cut). */
+export const MAX_TEXT_CHARS = 20_000;
+
+const isObj = (v: unknown): v is Record<string, any> => !!v && typeof v === 'object' && !Array.isArray(v);
+const intIn = (v: unknown, min: number, max: number) => Number.isInteger(v) && (v as number) >= min && (v as number) <= max;
+
+/** Problems with one vector provider (builder and runtime). */
+export function vectorProviderProblems(p: unknown): string[] {
+  if (!isObj(p)) return ['A vector provider is an object.'];
+  const what = `Vector provider ${typeof p.name === 'string' ? p.name : '?'}`;
+  const out: string[] = [];
+  if (typeof p.name !== 'string' || !PROVIDER_NAME.test(p.name)) out.push(`${what}: the name (static ID) is upper case letters, digits and _ (e.g. DOC_EMBEDDINGS).`);
+  if (!VECTOR_PROVIDER_TYPES.includes(p.type)) out.push(`${what}: the type is ai_service or function.`);
+  if (p.description !== undefined && p.description !== null && (typeof p.description !== 'string' || p.description.length > 500)) out.push(`${what}: the description is text (at most 500 characters).`);
+  if (!intIn(p.dimensions, 1, MAX_DIMENSIONS)) out.push(`${what}: the dimensions are a whole number from 1 to ${MAX_DIMENSIONS}.`);
+  if (p.type === 'ai_service') {
+    if (typeof p.service !== 'string' || !SERVICE_NAME.test(p.service)) out.push(`${what}: choose an AI service.`);
+    if (typeof p.model !== 'string' || !MODEL.test(p.model)) out.push(`${what}: the embedding model is its name, e.g. text-embedding-3-small or nomic-embed-text.`);
+    if (p.function !== undefined) out.push(`${what}: an AI service provider has no function.`);
+  }
+  if (p.type === 'function') {
+    if (typeof p.function !== 'string' || !FUNCTION_NAME.test(p.function)) out.push(`${what}: the function is schema.name, e.g. sales.embed.`);
+    if (p.service !== undefined || p.model !== undefined) out.push(`${what}: a function provider has no AI service or model.`);
+  }
+  if (p.batch_size !== undefined && !intIn(p.batch_size, 1, 2048)) out.push(`${what}: texts per request 1 to 2048.`);
+  if (p.per_minute !== undefined && !intIn(p.per_minute, 1, 10_000)) out.push(`${what}: requests per minute 1 to 10000.`);
+  return out;
+}
+
+/** Problems with an application's list: each entry, and names used twice. */
+export function vectorProvidersProblems(list: unknown): string[] {
+  if (!Array.isArray(list)) return ['Vector providers are a list.'];
+  const out: string[] = [];
+  const names = new Set<string>();
+  for (const p of list) {
+    out.push(...vectorProviderProblems(p));
+    if (isObj(p) && typeof p.name === 'string') {
+      if (names.has(p.name)) out.push(`Vector provider ${p.name} is defined twice.`);
+      names.add(p.name);
+    }
+  }
+  return out;
+}
+
+/** The application's vector providers (the owner reads them: the builder may have changed them a moment ago). */
+export async function loadVectorProviders(appId: number): Promise<VectorProvider[]> {
+  const r = await owner.one<{ vector_providers: unknown }>('select vector_providers from meta.app where id = $1', [appId]);
+  return Array.isArray(r?.vector_providers) ? (r!.vector_providers as VectorProvider[]) : [];
+}
+
+/** One valid provider of an application by its static ID, or an AiError. */
+export async function vectorProviderFor(appId: number, name: string): Promise<VectorProvider> {
+  const p = (await loadVectorProviders(appId)).find((x) => x.name === String(name ?? '').toUpperCase());
+  if (!p) throw new AiError('config', `There is no vector provider ${String(name ?? '').toUpperCase()} (Shared Components → Vector providers).`);
+  const problems = vectorProviderProblems(p);
+  if (problems.length) throw new AiError('config', problems.join(' '));
+  return p;
+}
+
+/** A vector as pgvector's text input: [0.1,0.2,…] (numbers only). */
+export function vectorLiteral(v: number[]): string {
+  if (!Array.isArray(v) || !v.length || v.some((x) => typeof x !== 'number' || !Number.isFinite(x))) throw new AiError('output', 'The embedding is not a list of numbers.');
+  return `[${v.join(',')}]`;
+}
+
+/** A client, a pool or pgkiln's Db: something that runs a query with parameters. */
+type Queryable = { query: (sql: string, params?: unknown[]) => Promise<{ rows: any[] }> };
+
+export interface EmbedInfo extends Pick<CallInfo, 'appId' | 'pageNo' | 'user' | 'debug'> {
+  appId: number;
+}
+
+/**
+ * Embed texts with a provider: one vector per text, each with the provider's dimensions. A function provider
+ * runs through meta.embed() on `client` (the caller's role and transaction); an AI service is called here.
+ */
+export async function embedTexts(p: VectorProvider, texts: string[], info: EmbedInfo, client?: Queryable): Promise<number[][]> {
+  if (!texts.length) return [];
+  const input = texts.map((t) => String(t ?? '').slice(0, MAX_TEXT_CHARS));
+  if (p.type === 'function') {
+    if (!client) throw new AiError('config', `Vector provider ${p.name}: a function provider needs a database connection.`);
+    const rows = (await client.query('select meta.embed(t, $2, $3) as v from unnest($1::text[]) with ordinality u(t, n) order by n', [input, p.name, info.appId])).rows;
+    return rows.map((r: { v: number[] | null }) => {
+      if (!Array.isArray(r.v)) throw new AiError('output', `Vector provider ${p.name}: the function returned no embedding.`);
+      return r.v.map(Number);
+    });
+  }
+  const { service: s, limits } = await serviceForApp(p.service!, info.appId);
+  if (s.provider === 'anthropic') throw new AiError('config', `AI service ${s.name} is a Claude service: Claude has no embeddings API (use an OpenAI, Gemini or OpenAI-compatible service).`);
+  const call: CallInfo = { ...info, source: 'embedding' };
+  await checkDailyLimit(s, limits, call, p.model!);
+  const client2 = new OpenAI({ apiKey: serviceKey(s), baseURL: baseUrlFor(s) || undefined, timeout: s.timeout_s * 1000, maxRetries: 1 });
+  const t0 = performance.now();
+  info.debug?.(6, `vector provider ${p.name}: ${input.length} text(s) to AI service ${s.name} (${s.provider}, ${p.model})`);
+  let tokens = 0;
+  try {
+    let res: OpenAI.CreateEmbeddingResponse;
+    try {
+      res = await client2.embeddings.create({
+        model: p.model!,
+        input,
+        encoding_format: 'float',
+        // OpenAI's text-embedding-3 models shorten their vectors to the dimensions asked for
+        ...(s.provider === 'openai' && /^text-embedding-3/.test(p.model!) ? { dimensions: p.dimensions } : {}),
+      });
+    } catch (e) {
+      throw openAiError(e);
+    }
+    tokens = res.usage?.prompt_tokens ?? 0;
+    const data = [...(res.data ?? [])].sort((a, b) => a.index - b.index);
+    if (data.length !== input.length) throw new AiError('output', `The AI service sent ${data.length} embeddings for ${input.length} texts.`);
+    const out = data.map((d) => (Array.isArray(d.embedding) ? d.embedding.map(Number) : []));
+    for (const v of out) {
+      if (v.length !== p.dimensions)
+        throw new AiError('output', `Vector provider ${p.name}: the model returned ${v.length} dimensions, the provider says ${p.dimensions} (change the provider's dimensions).`);
+      if (v.some((x) => !Number.isFinite(x))) throw new AiError('output', 'The embedding is not a list of numbers.');
+    }
+    await logUsage(call, { service: s, model: res.model || p.model!, inputTokens: tokens, outputTokens: 0, ms: performance.now() - t0, status: 'ok', message: null });
+    return out;
+  } catch (e) {
+    const err = e instanceof AiError ? e : new AiError('server', 'The embedding request failed.');
+    await logUsage(call, { service: s, model: p.model!, inputTokens: tokens, outputTokens: 0, ms: performance.now() - t0, status: 'error', message: err.kind + (err.status ? ` ${err.status}` : '') });
+    info.debug?.(1, `vector provider ${p.name}: ${err.kind}: ${err.message}`);
+    throw err;
+  }
+}
+
+// ---------------------------------------------------------------- the search text's embedding
+
+/** Search texts embedded recently (per server process, AI service providers only): a search as you type asks again. */
+const recent = new Map<string, { at: number; v: number[] }>();
+const RECENT_MAX = 500;
+const RECENT_MS = 10 * 60_000;
+
+/** The embedding of one search text (AI service providers: kept a few minutes in this process's memory). */
+export async function embedQuery(p: VectorProvider, text: string, info: EmbedInfo, client?: Queryable): Promise<number[]> {
+  if (p.type === 'function') return (await embedTexts(p, [text], info, client))[0];
+  const key = JSON.stringify([info.appId, p.name, p.service, p.model, p.dimensions, text]);
+  const hit = recent.get(key);
+  if (hit && Date.now() - hit.at < RECENT_MS) return hit.v;
+  const [v] = await embedTexts(p, [text], info, client);
+  if (recent.size >= RECENT_MAX) recent.delete(recent.keys().next().value!);
+  recent.set(key, { at: Date.now(), v });
+  return v;
+}
+
+/** Forget the remembered search texts (tests; a provider that changed). */
+export const forgetQueries = () => recent.clear();

package/src/ai/service.ts

@@ -99,7 +99,7 @@ export interface CallInfo {
   appId: number | null;
   pageNo?: number | null;
   user?: string | null;
-  source: 'process' | 'dynamic_action' | 'sql' | 'builder' | 'assistant' | 'nl2ir';
+  source: 'process' | 'dynamic_action' | 'sql' | 'builder' | 'assistant' | 'nl2ir' | 'embedding';
   /** debug messages of the request (level, text) */
   debug?: (level: number, text: string) => void;
 }
@@ -147,6 +147,17 @@ export async function serviceForApp(name: string, appId: number | null) {
   return { service: s, limits };
 }
 
+/** The application's daily request and token limits on a service (logged and refused when reached). */
+export async function checkDailyLimit(s: AiService, limits: { max_requests: number | null; max_tokens: string | null } | null, info: CallInfo, model = s.model) {
+  if (!limits || info.appId === null || (limits.max_requests === null && limits.max_tokens === null)) return;
+  const today = await usageToday(info.appId, s.id);
+  const over = (limits.max_requests !== null && today.requests >= limits.max_requests) || (limits.max_tokens !== null && today.tokens >= Number(limits.max_tokens));
+  if (over) {
+    await logUsage(info, { service: s, model, inputTokens: 0, outputTokens: 0, ms: 0, status: 'limited', message: 'daily limit' });
+    throw new AiError('limit', `This application has reached its daily limit for AI service ${s.name}: try again tomorrow.`);
+  }
+}
+
 /**
  * One AI request through a service: checks, daily limits, the call, the
  * usage log. A structured request returns the parsed JSON in `json`.
@@ -157,14 +168,7 @@ export async function generate(name: string, req: GenerateRequest, info: CallInf
   if (req.prompt.length > MAX_PROMPT_CHARS || (req.system?.length ?? 0) > MAX_SYSTEM_CHARS)
     throw new AiError('config', `The prompt is too long (at most ${MAX_PROMPT_CHARS} characters, the system prompt ${MAX_SYSTEM_CHARS}).`);
   if (!req.prompt.trim()) throw new AiError('config', 'The prompt is empty.');
-  if (limits && info.appId !== null && (limits.max_requests !== null || limits.max_tokens !== null)) {
-    const today = await usageToday(info.appId, s.id);
-    const over = (limits.max_requests !== null && today.requests >= limits.max_requests) || (limits.max_tokens !== null && today.tokens >= Number(limits.max_tokens));
-    if (over) {
-      await logUsage(info, { service: s, model: s.model, inputTokens: 0, outputTokens: 0, ms: 0, status: 'limited', message: 'daily limit' });
-      throw new AiError('limit', `This application has reached its daily limit for AI service ${s.name}: try again tomorrow.`);
-    }
-  }
+  await checkDailyLimit(s, limits, info);
   const provider = providerFor(s);
   const t0 = performance.now();
   info.debug?.(6, `AI service ${s.name} (${s.provider}, ${s.model})${req.schema ? ', structured output' : ''}`);

package/src/app.ts

@@ -26,6 +26,7 @@ import { diagnosticsRoutes } from './builder/diagnostics.ts';
 import { ldapRoutes } from './builder/ldap.ts';
 import { aiRoutes } from './builder/ai.ts';
 import { aiAgentRoutes } from './builder/ai-agents.ts';
+import { vectorProviderRoutes } from './builder/vector-providers.ts';
 import { mailRoutes } from './builder/mail.ts';
 import { backupRoutes } from './builder/backups.ts';
 import { availabilityRoutes } from './builder/availability.ts';
@@ -236,6 +237,8 @@ export async function buildApp(opts: { logger?: boolean } = {}) {
   await app.register(ldapRoutes);
   await app.register(aiRoutes);
   await app.register(aiAgentRoutes);
+  // (sprint 136, AP-1) vector providers: Shared Components and Workspace utilities
+  await app.register(vectorProviderRoutes);
   await app.register(mailRoutes);
   await app.register(backupRoutes);
   await app.register(availabilityRoutes);

package/src/builder/advisor.ts

@@ -11,8 +11,9 @@ import { handlerProblems } from '../runtime/rest.ts';
 import { plpgsqlBlock } from '../runtime/rest-handlers.ts';
 import { processProblems } from '../runtime/processes.ts';
 import { urlConfigProblem, urlTarget } from '../runtime/url-region.ts';
-import { configColumns, searchConfigProblems } from '../runtime/search.ts';
+import { configColumns, searchConfigProblems, textSearchConfig, tsvectorSql } from '../runtime/search.ts';
 import { COMPONENTS } from './components.ts';
+import { DISTANCES, PGVECTOR_MISSING, vectorColumnDims, vectorSchema, type Distance } from '../vector.ts';
 import { appEntries, type Entry } from './search.ts';
 import { appHeader, BASE, developer, region, send, shell, type Req } from './ui.ts';
 
@@ -240,10 +241,85 @@ export async function advise(appId: number, tx?: pg.ClientBase, opts: { db?: Db
         findings.push({ severity: 'error', entry, field: 'Source', message: problems.join(' ') });
         continue;
       }
-      const cols = configColumns(row).map((x) => `t.${pg.escapeIdentifier(x)}`).join(', ') || '*';
+      let cols = configColumns(row).map((x) => `t.${pg.escapeIdentifier(x)}`).join(', ') || '*';
+      // (135) a full-text configuration: its tsvector matched with its text search configuration
+      const fulltext = row.search_type === 'fulltext' || row.search_type === 'hybrid';
+      // (136) a vector or hybrid configuration also reads its vector (and key) column
+      const vectorType = row.search_type === 'vector' || row.search_type === 'hybrid';
+      if (vectorType) cols = [cols === '*' ? 't.*' : cols, ...[row.vector_column, row.key_column].filter(Boolean).map((x: string) => `t.${pg.escapeIdentifier(x)}`)].join(', ');
+      if (fulltext) {
+        const cfg = await textSearchConfig(c, row.fts_config, app.language);
+        if (row.fts_config && cfg !== row.fts_config)
+          findings.push({ severity: 'error', entry, field: 'Text search configuration', message: `Text search configuration ${row.fts_config} doesn't exist (pg_ts_config): ${cfg} is used instead.` });
+        const lit = `${pg.escapeLiteral(cfg)}::regconfig`;
+        // (ts_rank_cd takes only a tsvector: text @@ tsquery would pass)
+        cols = `${cols === '*' ? 't.*' : cols}, ts_rank_cd(${tsvectorSql(row, lit)}, websearch_to_tsquery(${lit}, 'x'))`;
+      }
       checked++;
       const problem = await checkSql(c, `select ${cols} from ${row.source_type === 'table' ? row.source.trim() : `(${row.source})`} t`, 'select');
-      if (problem) findings.push({ ...problem, entry, field: 'Source' });
+      if (problem) findings.push({ ...problem, entry, field: fulltext ? 'Source / tsvector' : 'Source' });
+      // (135) a full-text search on a table without a GIN index on its tsvector reads every row
+      if (fulltext && !problem && row.source_type === 'table') {
+        const vector = String(row.fts_vector ?? '').trim();
+        const column = /^[A-Za-z_][A-Za-z0-9_$]{0,62}$/.test(vector) ? vector : null;
+        await c.query('savepoint advisor_fts');
+        const ix = (await c.query<{ checkable: boolean; gin: boolean }>(
+          `select cl.relkind in ('r', 'p', 'm') as checkable,
+                  exists (select from pg_catalog.pg_index i join pg_catalog.pg_class ic on ic.oid = i.indexrelid join pg_catalog.pg_am am on am.oid = ic.relam
+                           where i.indrelid = cl.oid and am.amname = 'gin'
+                             and case when $2::text is not null
+                                      then (select a.attnum from pg_catalog.pg_attribute a where a.attrelid = cl.oid and a.attname = $2 and not a.attisdropped) = any (i.indkey::int2[])
+                                      else i.indexprs is not null and pg_catalog.pg_get_indexdef(i.indexrelid) ~* 'to_tsvector' end) as gin
+             from pg_catalog.pg_class cl where cl.oid = $1::regclass`, [row.source.trim(), column]).catch(async () => {
+          await c.query('rollback to savepoint advisor_fts');
+          return { rows: [] };
+        })).rows[0];
+        if (ix?.checkable && (!ix.gin || !vector))
+          findings.push({
+            severity: 'warning', entry, field: 'tsvector column or expression',
+            message: column
+              ? `${row.source.trim()}.${column} has no GIN index: every search reads the whole table (create index … on ${row.source.trim()} using gin (${pg.escapeIdentifier(column)})).`
+              : vector
+                ? `${row.source.trim()} has no GIN index on a to_tsvector expression: every search reads the whole table.`
+                : `The tsvector is built from the searchable columns on every search, so no index can help: add a tsvector column with a GIN index (SQL Workshop › Table Properties › Add full-text search) and name it here.`,
+          });
+      }
+    }
+    // (136) vector and hybrid configurations: pgvector, the provider, the column's dimensions and its index
+    for (const { kind, row } of all.filter((x) => x.kind === 'search_config' && (x.row.search_type === 'vector' || x.row.search_type === 'hybrid') && !searchConfigProblems(x.row).length)) {
+      const entry = byKey.get(`${kind}-${row.id}`) ?? null;
+      const schema = await vectorSchema(c);
+      if (!schema) {
+        findings.push({ severity: 'error', entry, field: 'Search type', message: PGVECTOR_MISSING });
+        continue;
+      }
+      const provider = (Array.isArray(app.vector_providers) ? app.vector_providers : []).find((p: { name: string }) => p.name === row.vector_provider);
+      if (!provider) findings.push({ severity: 'error', entry, field: 'Vector provider', message: `There is no vector provider ${row.vector_provider} (Shared Components → Vector providers).` });
+      if (row.source_type !== 'table') continue;
+      await c.query('savepoint advisor_vec');
+      const info = await (async () => {
+        const dims = await vectorColumnDims(c, row.source.trim(), row.vector_column);
+        const ix = (await c.query<{ ok: boolean; ops: string | null }>(
+          `select exists (select from pg_catalog.pg_index i join pg_catalog.pg_class ic on ic.oid = i.indexrelid join pg_catalog.pg_am am on am.oid = ic.relam
+                           where i.indrelid = $1::regclass and am.amname in ('hnsw', 'ivfflat')
+                             and (select a.attnum from pg_catalog.pg_attribute a where a.attrelid = $1::regclass and a.attname = $2) = any (i.indkey::int2[])) as ok,
+                  (select string_agg(o.opcname, ',') from pg_catalog.pg_index i join pg_catalog.pg_opclass o on o.oid = any (i.indclass::oid[])
+                    where i.indrelid = $1::regclass and (select a.attnum from pg_catalog.pg_attribute a where a.attrelid = $1::regclass and a.attname = $2) = any (i.indkey::int2[])) as ops`,
+          [row.source.trim(), row.vector_column])).rows[0];
+        return { dims, ix };
+      })().catch(async () => {
+        await c.query('rollback to savepoint advisor_vec');
+        return null;
+      });
+      if (!info) continue;
+      if (info.dims === undefined) findings.push({ severity: 'error', entry, field: 'Vector column', message: `${row.source.trim()} has no column ${row.vector_column}.` });
+      else if (info.dims === null) findings.push({ severity: 'error', entry, field: 'Vector column', message: `${row.source.trim()}.${row.vector_column} is not a vector(n) column.` });
+      else if (provider && info.dims !== provider.dimensions)
+        findings.push({ severity: 'error', entry, field: 'Vector column', message: `${row.source.trim()}.${row.vector_column} is vector(${info.dims}), vector provider ${provider.name} gives ${provider.dimensions} dimensions.` });
+      if (info.dims && !info.ix?.ok)
+        findings.push({ severity: 'warning', entry, field: 'Vector column', message: `${row.source.trim()}.${row.vector_column} has no HNSW index: every search reads the whole table (SQL Workshop › Table Properties › Add vector column adds one).` });
+      else if (info.dims && info.ix?.ops && !info.ix.ops.split(',').includes(DISTANCES[(row.distance ?? 'cosine') as Distance].opclass))
+        findings.push({ severity: 'warning', entry, field: 'Distance', message: `The index on ${row.vector_column} is for ${info.ix.ops}, not ${DISTANCES[(row.distance ?? 'cosine') as Distance].opclass}: the ${row.distance ?? 'cosine'} search can't use it.` });
     }
     // the search regions name existing configurations
     for (const { kind, row } of all.filter((x) => x.kind === 'region' && x.row.type === 'search')) {

package/src/builder/ai-agents.ts

@@ -24,6 +24,7 @@ const TOOL_TYPES: [string, string][] = [
   ['rest', 'REST data source'],
   ['javascript', 'JavaScript action in the browser'],
   ['navigate', 'Navigate: a link to a page'],
+  ['retrieve', 'Retrieve (RAG): rows of a search configuration'],
 ];
 
 const when = (d: Date | string) => new Date(d).toISOString().slice(0, 19).replace('T', ' ');
@@ -87,6 +88,7 @@ async function agentForm(a: any, s: Session, x: Partial<AiAgent>, original: stri
 
 async function toolForm(a: any, s: Session, x: Partial<AiTool> & { parameters_text?: string }, original: string, problems: string[]): Promise<Raw> {
   const sources = (await owner.query<{ name: string }>('select name from meta.rest_source where app_id = $1 order by name', [a.id])).rows.map((r) => r.name);
+  const configs = (await owner.query<{ name: string; search_type: string }>(`select name, search_type from meta.search_config where app_id = $1 and source_type <> 'list' order by name`, [a.id])).rows;
   const params = x.parameters_text ?? (x.parameters ? JSON.stringify(x.parameters, null, 2) : '');
   const usedBy = original ? ((a.ai_agents ?? []) as AiAgent[]).filter((g) => (g.tools ?? []).includes(original)).map((g) => g.name) : [];
   return html`${problems.length ? html`<div class="alert alert-error" role="alert">${problems.join(' ')}</div>` : ''}
@@ -100,7 +102,10 @@ async function toolForm(a: any, s: Session, x: Partial<AiTool> & { parameters_te
         ${select('source', 'REST data source (REST)', x.source ?? '', [['', '-'], ...sources.map((n): [string, string] => [n, n])], 'The parameters become the source\'s parameters.')}
         ${input('action', 'Action (JavaScript)', x.action ?? '', { placeholder: 'e.g. highlightOrder', help: 'After the answer the browser calls the function registered with pgkiln.actions.register(name) with {value: the arguments}, and the dynamic actions of the custom event of this name.' })}
         ${input('pages', 'Pages (Navigate)', (x.pages ?? []).join(', '), { placeholder: 'e.g. 3, 7', help: 'The page numbers the agent may link to. pgkiln offers only those the user may open and builds the URL (item values with their checksum).' })}
-        ${area('parameters', 'Parameters (JSON; not for Navigate)', params, 6, '{"DNAME": {"type": "string", "description": "A department name"}, "LIMIT": {"type": "integer", "optional": true}}. Types: string, integer, number, boolean, date; "enum" for a list of texts.', true)}
+        ${select('search_config', 'Search configuration (Retrieve)', x.search_config ?? '', [['', '-'], ...configs.map((c): [string, string] => [c.name, `${c.name} (${c.search_type})`]),
+          ...(x.search_config && !configs.some((c) => c.name === x.search_config) ? [[x.search_config, `${x.search_config} (does not exist)`] as [string, string]] : [])],
+          'The model gives a text; pgkiln runs this search configuration (best: vector or hybrid) as the application\'s role, so row level security applies, and returns the best rows\' title, text and score.')}
+        ${area('parameters', 'Parameters (JSON; not for Navigate or Retrieve)', params, 6, '{"DNAME": {"type": "string", "description": "A department name"}, "LIMIT": {"type": "integer", "optional": true}}. Types: string, integer, number, boolean, date; "enum" for a list of texts.', true)}
         ${input('max_rows', 'Maximum rows', x.max_rows ?? '', { type: 'number', placeholder: '50', help: 'Rows the model gets back (1 – 500).' })}
         ${input('timeout_s', 'Time limit (seconds)', x.timeout_s ?? '', { type: 'number', placeholder: '15', help: 'SQL and function tools (1 – 60).' })}
         ${authzSelect(await schemes(a.id), x.authz, 'Only users who pass it are offered the tool.')}
@@ -151,8 +156,9 @@ function toolFrom(b: Body, errors: string[]): AiTool & { parameters_text?: strin
   if (type === 'rest') x.source = one(b.source).trim().toUpperCase();
   if (type === 'javascript') x.action = one(b.action).trim();
   if (type === 'navigate') x.pages = one(b.pages).split(/[\s,;]+/).filter(Boolean).map(Number);
+  if (type === 'retrieve') x.search_config = one(b.search_config).trim().toUpperCase();
   const params = one(b.parameters).trim();
-  if (params && type !== 'navigate') {
+  if (params && type !== 'navigate' && type !== 'retrieve') {
     try {
       x.parameters = JSON.parse(params);
     } catch {
@@ -160,11 +166,11 @@ function toolFrom(b: Body, errors: string[]): AiTool & { parameters_text?: strin
       x.parameters_text = params;
     }
   }
-  if (type === 'sql' || type === 'function' || type === 'rest') {
+  if (type === 'sql' || type === 'function' || type === 'rest' || type === 'retrieve') {
     const rows = intOrUndef(one(b.max_rows), 'Maximum rows', errors);
     if (rows !== undefined) x.max_rows = rows;
   }
-  if (type === 'sql' || type === 'function') {
+  if (type === 'sql' || type === 'function' || type === 'retrieve') {
     const t = intOrUndef(one(b.timeout_s), 'Time limit', errors);
     if (t !== undefined) x.timeout_s = t;
   }
@@ -225,7 +231,7 @@ export async function aiAgentRoutes(app: FastifyInstance) {
     const problems = definitionProblems(agents, tools);
     const main = html`${appHeader(a, 'shared', s)}
       <div class="title-row"><h1>AI Agents</h1></div>
-      <p class="muted u-mt0">An agent answers in an AI assistant region and acts only through the tools you give it: SQL queries (read-only, as the application's database role, so row level security applies), PL/pgSQL functions, REST data sources, JavaScript actions in the browser, and links to pages (pgkiln builds the URL and its checksum, only for pages the user may open). What users type and the tools' results go to the agent's AI service. <a href="${BASE}/apps/${a.id}/ai">AI usage</a> lists every tool call.</p>
+      <p class="muted u-mt0">An agent answers in an AI assistant region and acts only through the tools you give it: SQL queries (read-only, as the application's database role, so row level security applies), PL/pgSQL functions, REST data sources, JavaScript actions in the browser, and links to pages (pgkiln builds the URL and its checksum, only for pages the user may open). A Retrieve tool gives the agent the rows of a search configuration that match a text by meaning (RAG), again as the application's role. What users type and the tools' results go to the agent's AI service. <a href="${BASE}/apps/${a.id}/ai">AI usage</a> lists every tool call.</p>
       ${problems.length ? html`<div class="alert alert-error" role="alert">${problems.join(' ')}</div>` : ''}
       <div class="columns">
         ${region('Agents', html`<div class="table-wrap"><table class="report report-reflow"><thead><tr><th>Agent</th><th>AI service</th><th>Tools</th><th>Authorization</th></tr></thead>
@@ -322,6 +328,8 @@ export async function aiAgentRoutes(app: FastifyInstance) {
       const out = aiToolProblems(tool);
       if (tool.type === 'rest' && tool.source && !(await owner.one('select 1 from meta.rest_source where app_id = $1 and name = $2', [a.id, tool.source])))
         out.push(`There is no REST data source ${tool.source}.`);
+      if (tool.type === 'retrieve' && tool.search_config && !(await owner.one(`select 1 from meta.search_config where app_id = $1 and name = $2 and source_type <> 'list'`, [a.id, tool.search_config])))
+        out.push(`There is no search configuration ${tool.search_config} with a table or SQL source.`);
       if (tool.type === 'navigate') {
         const known = new Set((await owner.query<{ page_no: number }>('select page_no from meta.page where app_id = $1', [a.id])).rows.map((p) => p.page_no));
         for (const p of tool.pages ?? []) if (!known.has(p)) out.push(`There is no page ${p}.`);

package/src/builder/components.ts

@@ -722,8 +722,21 @@ export const COMPONENTS: Record<string, ComponentSpec> = {
     plural: 'Search configurations',
     icon: 'search',
     summary: (c) => `${c.name} (${c.source_type})`,
-    defaults: { source_type: 'table', seq: 10, max_results: 10, searchable_columns: [] },
+    defaults: { source_type: 'table', search_type: 'standard', seq: 10, max_results: 10, searchable_columns: [], did_you_mean: false, distance: 'cosine' },
     validate: (v) => searchConfigProblems(v as never).join(' ') || null,
+    // (135) empty means: built from the searchable columns / the application language's configuration
+    beforeSave: async (v) => {
+      for (const k of ['fts_vector', 'fts_config'] as const) if (typeof v[k] === 'string' && !(v[k] as string).trim()) v[k] = null;
+      if (typeof v.fts_config === 'string') v.fts_config = v.fts_config.trim().toLowerCase();
+      // (136) vector and hybrid search
+      for (const k of ['vector_column', 'key_column'] as const) if (typeof v[k] === 'string') v[k] = (v[k] as string).trim() || null;
+      if (typeof v.vector_provider === 'string') v.vector_provider = v.vector_provider.trim().toUpperCase() || null;
+      if (typeof v.min_similarity === 'string') {
+        const t = v.min_similarity.trim().replace(',', '.');
+        if (t && !Number.isFinite(Number(t))) throw new Error('Minimum similarity must be a number from -1 to 1.');
+        v.min_similarity = t ? Number(t) : null;
+      }
+    },
     fields: [
       { name: 'name', label: 'Name (static ID)', kind: 'upper', group: 'Identification', help: 'e.g. CUSTOMERS; a search region names it in {"configs": ["CUSTOMERS"]}.' },
       { name: 'label', label: 'Label', kind: 'text', group: 'Identification', help: 'The heading of its results, e.g. Customers.' },
@@ -733,6 +746,24 @@ export const COMPONENTS: Record<string, ComponentSpec> = {
       { name: 'source', label: 'Source', kind: 'code', wide: true, group: 'Source', help: 'table: sales.customers · sql: select id, name, city from sales.customers where active · list: the list\'s name.' },
       { name: 'searchable_columns', label: 'Searchable columns', kind: 'list', group: 'Source',
         help: 'Comma-separated columns the search text is looked for in (each word must be found in one of them). Empty: the title, subtitle and description columns.' },
+      { name: 'search_type', label: 'Search type', kind: 'select', options: ['standard', 'fulltext', 'vector', 'hybrid'], group: 'Search',
+        help: 'standard: each word anywhere in one of the searchable columns (case-insensitive) · fulltext: PostgreSQL full-text search: stemmed words ("running" finds "run"), "quoted phrases", or, -word; the best match first, the matched words highlighted (table, sql) · vector: by meaning: the rows whose embedding is nearest to the search text\'s (pgvector) · hybrid: full-text and vector together (reciprocal rank fusion).' },
+      { name: 'fts_vector', label: 'tsvector column or expression', kind: 'code', wide: true, group: 'Search',
+        help: 'fulltext: a tsvector column of the source (e.g. search_vector, with a GIN index: SQL Workshop › Table Properties › Add full-text search), or an expression, e.g. to_tsvector(\'english\', title || \' \' || body). Empty: to_tsvector over the searchable columns (no index can help).' },
+      { name: 'fts_config', label: 'Text search configuration', kind: 'text', group: 'Search',
+        help: 'fulltext: english, dutch, german, french, simple … (pg_ts_config); the same one the tsvector was made with. Empty: the application\'s primary language\'s, else simple.' },
+      { name: 'did_you_mean', label: 'Did you mean', kind: 'bool', group: 'Search',
+        help: 'With the pg_trgm extension installed: when nothing is found, suggest the closest words from the searched columns (reads up to 10,000 rows). Without pg_trgm nothing is suggested.' },
+      { name: 'vector_provider', label: 'Vector provider', kind: 'upper', group: 'Vector search',
+        help: 'vector, hybrid: the static ID of a vector provider (Shared Components → Vector providers) that embeds the search text, the same one that embedded the rows.' },
+      { name: 'vector_column', label: 'Vector column', kind: 'text', group: 'Vector search',
+        help: 'vector, hybrid: the source\'s vector(n) column, e.g. embedding (SQL Workshop › Table Properties › Add vector column adds it, with an HNSW index, and keeps it current); n must be the provider\'s dimensions.' },
+      { name: 'key_column', label: 'Key column', kind: 'text', group: 'Vector search',
+        help: 'hybrid: the primary key (e.g. id) that matches the full-text and the vector results. The full-text side uses the tsvector and text search configuration above.' },
+      { name: 'distance', label: 'Distance', kind: 'select', options: ['cosine', 'l2', 'inner_product'], group: 'Vector search',
+        help: 'cosine (most embedding models), l2 (Euclidean) or inner_product; the HNSW index must use the same (vector_cosine_ops, vector_l2_ops, vector_ip_ops).' },
+      { name: 'min_similarity', label: 'Minimum similarity', kind: 'text', group: 'Vector search',
+        help: 'vector, hybrid: leave out rows less similar than this (cosine: 1 − distance, e.g. 0.3; inner product: the product; l2: 1 / (1 + distance)). Empty: the nearest rows, however far.' },
       { name: 'title_column', label: 'Title column', kind: 'text', group: 'Result', help: 'Required (table, sql); results are ordered by it.' },
       { name: 'subtitle_column', label: 'Subtitle column', kind: 'text', group: 'Result' },
       { name: 'description_column', label: 'Description column', kind: 'text', group: 'Result' },