pgkiln 0.74.0 → 0.76.0

package/src/builder/json-sources.ts

@@ -0,0 +1,395 @@
+import type { FastifyInstance, FastifyReply } from 'fastify';
+import { joinStatements, recordChanges } from '../changelog.ts';
+import { owner } from '../db.ts';
+import { html, raw, type Raw } from '../html.ts';
+import { icon } from '../icons.ts';
+import {
+  JSON_TYPES, POSITION, ROW_KEY, ident, jsonSourceDdl, jsonSourceProblems, primaryKey, proposeProfile, sampleDocuments, tableGrants,
+  type JsonArray, type JsonColumn, type JsonSource, type JsonType,
+} from '../json-sources.ts';
+import type { Session } from '../session.ts';
+import { clientIp, logActivity } from '../session.ts';
+import { clearCompletions } from './code-editor.ts';
+import { appOr404 } from './forms.ts';
+import { appHeader, back, BASE, csrf, developer, flash, input, region, send, shell, type Req } from './ui.ts';
+import { currentWorkspace, usableSchemas, workshopDb } from './workspaces.ts';
+
+// (sprint 142, AP-8; APEX: Shared Components → Data Sources → JSON Sources)
+//
+// Shared Components → JSON sources: the application's JSON sources, a JSON list in meta.app (json_sources,
+// migration 142) exported with the application. A source is a table's jsonb column with a data profile, proposed
+// from sample rows (Propose from the data) and edited here: a column per path (name, type, read only), arrays of
+// objects as views of their own. Create views makes the views (src/json-sources.ts) with the developer's SQL
+// Workshop connection in one transaction, recorded in the Change Log; Create page makes a form on the view with a
+// detail grid per array. The table, its key and its columns are read from the catalog with that connection: only
+// tables in the schemas it may use, and only names found there reach the DDL (quoted).
+
+type Body = Record<string, string | string[] | undefined>;
+type Db = Awaited<ReturnType<typeof workshopDb>>;
+const one = (v: string | string[] | undefined) => (Array.isArray(v) ? v[0] : v) ?? '';
+const SYSTEM = ['pg_catalog', 'information_schema', 'meta'];
+
+const crumbs = (a: { id: number; name: string }, last?: string): [string, string?][] => [
+  ['App Builder', BASE], [a.name, `${BASE}/apps/${a.id}`], ['Shared Components', `${BASE}/apps/${a.id}/shared`],
+  ...(last ? [['JSON sources', `${BASE}/apps/${a.id}/json-sources`], [last]] as [string, string?][] : [['JSON sources']] as [string, string?][]),
+];
+const listOf = (a: any) => (Array.isArray(a.json_sources) ? a.json_sources : []) as JsonSource[];
+const srcUrl = (appId: number, name: string) => `${BASE}/apps/${appId}/json-sources/source?name=${encodeURIComponent(name)}`;
+const pathText = (p: string[]) => p.join('.');
+const pathFrom = (t: string) => t.trim() === '' || t.trim() === '.' ? [] : t.split('.').map((k) => k.trim());
+
+/** The tables (and their json / jsonb columns) the developer's connection may use. */
+async function jsonColumns(db: Db) {
+  const schemas = (await usableSchemas(db)).filter((n) => !SYSTEM.includes(n));
+  if (!schemas.length) return [];
+  return (await db.query<{ schema: string; table: string; column: string }>(
+    `select n.nspname as schema, c.relname as table, a.attname as column
+       from pg_class c join pg_namespace n on n.oid = c.relnamespace join pg_attribute a on a.attrelid = c.oid
+      where c.relkind in ('r', 'p') and a.attnum > 0 and not a.attisdropped and a.atttypid in ('jsonb'::regtype, 'json'::regtype)
+        and n.nspname = any ($1) and has_table_privilege(c.oid, 'select')
+      order by 1, 2, 3`, [schemas])).rows;
+}
+
+/** Which of the source's views exist now (and whether they are this source's: their trigger). */
+async function viewState(db: Db, x: JsonSource) {
+  const names = [x.view, ...x.arrays.map((y) => y.view)];
+  const rows = (await db.query<{ n: string; ours: boolean }>(
+    `select c.relname as n, exists (select from pg_trigger t where t.tgrelid = c.oid and t.tgname = 'json_source_write') as ours
+       from pg_class c join pg_namespace s on s.oid = c.relnamespace where s.nspname = $1 and c.relname = any ($2)`, [x.schema, names])).rows;
+  return new Map(rows.map((r) => [r.n, r.ours]));
+}
+
+function columnsTable(prefix: string, cols: JsonColumn[], extra: number, emptyPath: boolean): Raw {
+  const rows = [...cols, ...Array.from({ length: extra }, (): JsonColumn => ({ name: '', path: [], type: 'text' }))];
+  return html`<div class="table-wrap"><table class="report js-cols"><thead><tr><th scope="col">Column</th><th scope="col">Path</th><th scope="col">Type</th><th scope="col">Read only</th></tr></thead><tbody>
+    ${rows.map((c, i) => html`<tr>
+      <td><input name="${prefix}.${i}.name" value="${c.name}" class="code" aria-label="Column ${i + 1}: name" autocomplete="off" spellcheck="false"></td>
+      <td><input name="${prefix}.${i}.path" value="${pathText(c.path)}" class="code" aria-label="Column ${i + 1}: path" autocomplete="off" spellcheck="false"${emptyPath ? raw(' placeholder="(the element itself)"') : ''}></td>
+      <td><select name="${prefix}.${i}.type" aria-label="Column ${i + 1}: type">${JSON_TYPES.map((t) => html`<option${t === c.type ? raw(' selected') : ''}>${t}</option>`)}</select></td>
+      <td class="te-c"><input type="checkbox" name="${prefix}.${i}.readonly" aria-label="Column ${i + 1}: read only"${c.readonly ? raw(' checked') : ''}></td></tr>`)}
+    </tbody></table></div>`;
+}
+
+function sourceForm(a: any, s: Session, x: JsonSource, original: string, problems: string[], tables: { schema: string; table: string; column: string }[]): Raw {
+  const opts = tables.map((t) => `${t.schema}.${t.table}.${t.column}`);
+  const cur = x.schema ? `${x.schema}.${x.table}.${x.column}` : '';
+  if (cur && !opts.includes(cur)) opts.unshift(cur);
+  return html`${problems.length ? html`<div class="alert alert-error" role="alert"><ul>${problems.map((p) => html`<li>${p}</li>`)}</ul></div>` : ''}
+    <form method="post" action="${BASE}/apps/${a.id}/json-sources/source">${csrf(s)}<input type="hidden" name="original" value="${original}">
+      <fieldset class="prop-group"><legend>JSON source</legend><div class="form-grid">
+        ${input('name', 'Name (static ID)', x.name, { required: true, placeholder: 'e.g. EMP_PROFILE', help: 'Upper case letters, digits and _.' })}
+        ${input('description', 'Description', x.description ?? '')}
+        <div class="field"><label class="label" for="js-source">Table and jsonb column</label>
+          <select id="js-source" name="source">${opts.length ? opts.map((o) => html`<option${o === cur ? raw(' selected') : ''}>${o}</option>`) : html`<option value="">(no table with a json or jsonb column)</option>`}</select>
+          <small class="help">A column of a table, or the document column of a table of documents. The table needs a primary key.</small></div>
+        ${input('view', 'View', x.view, { required: true, placeholder: 'e.g. emp_profile', help: 'Made in the table\'s schema. Regions, REST and MCP use it like a table.' })}
+        <div class="field"><label class="check"><input type="checkbox" name="rows"${x.rows ? raw(' checked') : ''}> The view also adds and deletes rows of the table</label>
+          <small class="help">For a table of documents: an insert into the view adds a row, a delete removes it. Off: the view only changes the documents of existing rows.</small></div>
+      </div></fieldset>
+      <fieldset class="prop-group"><legend>Columns</legend>
+        <p class="muted u-mt0">A column per path in the document (keys separated by dots, e.g. <code>address.city</code>). A value that doesn't read as the type shows empty. Saving a row writes only the paths whose values changed: keys this list doesn't name stay in the document. Clear a name to remove a column.</p>
+        ${columnsTable('c', x.columns, 3, false)}</fieldset>
+      ${x.arrays.map((y, j) => html`<fieldset class="prop-group"><legend>Array ${y.name}</legend><div class="form-grid">
+          ${input(`a.${j}.name`, 'Name', y.name, { required: true })}
+          ${input(`a.${j}.path`, 'Path', pathText(y.path), { required: true, help: 'The array in the document, e.g. skills.' })}
+          ${input(`a.${j}.view`, 'View', y.view, { required: true, help: `One row per element: the table's key, ${ROW_KEY} (the element's key) and ${POSITION} (1, 2 …), then these columns. Use it as a detail grid.` })}
+          <div class="field"><label class="check"><input type="checkbox" name="a.${j}.drop"> Remove this array</label></div>
+        </div>${columnsTable(`a.${j}.c`, y.columns, 2, true)}</fieldset>`)}
+      <fieldset class="prop-group"><legend>Add an array</legend><div class="form-grid">
+        ${input(`a.${x.arrays.length}.name`, 'Name', '', { placeholder: 'e.g. skills' })}
+        ${input(`a.${x.arrays.length}.path`, 'Path', '', { placeholder: 'e.g. skills' })}
+        ${input(`a.${x.arrays.length}.view`, 'View', '', { placeholder: 'e.g. emp_profile_skills' })}
+      </div><small class="help">Add the array, then Propose from the data fills its columns (or type them after saving).</small></fieldset>
+      <div class="buttons">
+        <button class="btn btn-hot" name="mode" value="save">${original ? 'Save' : 'Add JSON source'}</button>
+        <button class="btn" name="mode" value="propose" formnovalidate>${icon('lightbulb')} Propose from the data</button>
+      </div>
+    </form>`;
+}
+
+/** The definition from the posted form (names trimmed; empty column rows left out). */
+function sourceFrom(b: Body, key: string[]): JsonSource {
+  const [schema = '', table = '', column = ''] = (() => {
+    const v = one(b.source);
+    const i = v.lastIndexOf('.');
+    const j = v.indexOf('.');
+    return i > j && j > 0 ? [v.slice(0, j), v.slice(j + 1, i), v.slice(i + 1)] : [];
+  })();
+  const colsFrom = (prefix: string) => {
+    const out: JsonColumn[] = [];
+    for (let i = 0; i < 400 && b[`${prefix}.${i}.name`] !== undefined; i++) {
+      const name = one(b[`${prefix}.${i}.name`]).trim();
+      if (!name) continue;
+      const type = one(b[`${prefix}.${i}.type`]) as JsonType;
+      const c: JsonColumn = { name, path: pathFrom(one(b[`${prefix}.${i}.path`])), type };
+      if (one(b[`${prefix}.${i}.readonly`]) === 'on') c.readonly = true;
+      out.push(c);
+    }
+    return out;
+  };
+  const arrays: JsonArray[] = [];
+  for (let j = 0; j < 50 && b[`a.${j}.name`] !== undefined; j++) {
+    const name = one(b[`a.${j}.name`]).trim();
+    if (!name || one(b[`a.${j}.drop`]) === 'on') continue;
+    arrays.push({ name, path: pathFrom(one(b[`a.${j}.path`])), view: one(b[`a.${j}.view`]).trim(), columns: colsFrom(`a.${j}.c`) });
+  }
+  const x: JsonSource = { name: one(b.name).trim().toUpperCase(), schema, table, key, column, view: one(b.view).trim(), columns: colsFrom('c'), arrays };
+  const description = one(b.description).trim().slice(0, 500);
+  if (description) x.description = description;
+  if (one(b.rows) === 'on') x.rows = true;
+  return x;
+}
+
+async function update(appId: number, change: (list: JsonSource[]) => string[]) {
+  return owner.tx(async (c) => {
+    const row = (await c.query<{ json_sources: JsonSource[] }>('select json_sources from meta.app where id = $1 for update', [appId])).rows[0];
+    const list = [...(row?.json_sources ?? [])];
+    const problems = change(list);
+    if (problems.length) return problems;
+    await c.query('update meta.app set json_sources = $2, updated_at = now() where id = $1', [appId, JSON.stringify(list)]);
+    return [];
+  });
+}
+
+/** Item type of a column on the generated form. */
+const itemType = (t: JsonType) => (({ integer: 'number', bigint: 'number', numeric: 'number', boolean: 'switch', date: 'date', timestamp: 'datetime', timestamptz: 'datetime', jsonb: 'textarea', text: 'text' }) as const)[t];
+const label = (n: string) => n.replace(/_/g, ' ').replace(/^./, (c) => c.toUpperCase());
+
+/** A form on the source's view with a detail grid per array; returns the page number. */
+export async function createSourcePage(appId: number, x: JsonSource, pageNo: number, title: string) {
+  if (x.key.length !== 1) throw new Error('Create page needs a table with a one-column key; use the views in regions of your own.');
+  const key = x.key[0];
+  const keyItem = `P${pageNo}_${key.toUpperCase()}`;
+  return owner.tx(async (c) => {
+    if ((await c.query('select 1 from meta.page where app_id = $1 and page_no = $2', [appId, pageNo])).rows.length) throw new Error(`Page ${pageNo} exists already.`);
+    const pageId = (await c.query<{ id: number }>('insert into meta.page (app_id, page_no, name, title) values ($1, $2, $3, $3) returning id', [appId, pageNo, title])).rows[0].id;
+    const form = (await c.query<{ id: number }>(
+      `insert into meta.region (page_id, seq, title, type, table_name, pk_column, pk_item) values ($1, 10, $2, 'form', $3, $4, $5) returning id`,
+      [pageId, title, `${x.schema}.${x.view}`, key, keyItem])).rows[0].id;
+    await c.query(`insert into meta.item (page_id, region_id, seq, name, label, type, source_column) values ($1, $2, 10, $3, $4, 'hidden', $5)`, [pageId, form, keyItem, label(key), key]);
+    let seq = 20;
+    for (const col of x.columns) {
+      await c.query(`insert into meta.item (page_id, region_id, seq, name, label, type, source_column, readonly_condition) values ($1, $2, $3, $4, $5, $6, $7, $8)`,
+        [pageId, form, seq, `P${pageNo}_${col.name.toUpperCase()}`.slice(0, 60), label(col.name), itemType(col.type), col.name, col.readonly ? 'true' : null]);
+      seq += 10;
+    }
+    await c.query(`insert into meta.button (page_id, region_id, seq, name, label, action) values ($1, $2, 10, 'SAVE', 'Apply Changes', 'submit')`, [pageId, form]);
+    await c.query(`insert into meta.process (page_id, region_id, seq, name, type, point) values ($1, $2, 10, $3, 'form_dml', 'submit')`, [pageId, form, `Process form ${title}`]);
+    let rseq = 20;
+    for (const y of x.arrays) {
+      const cols = [ident(key), ROW_KEY, POSITION, ...y.columns.map((col) => ident(col.name))];
+      const grid = (await c.query<{ id: number }>(
+        `insert into meta.region (page_id, seq, title, type, table_name, pk_column, source, config) values ($1, $2, $3, 'grid', $4, $5, $6, $7) returning id`,
+        [pageId, rseq, label(y.name), `${x.schema}.${y.view}`, ROW_KEY,
+          `select ${cols.join(', ')}\n  from ${ident(x.schema)}.${ident(y.view)}\n where ${ident(key)}::text = :${keyItem}\n order by ${POSITION}`,
+          JSON.stringify({ master: { item: keyItem, column: key }, readonly: [POSITION], layout: { hidden: [key, ROW_KEY] }, headings: { [POSITION]: '#' }, row_actions: { delete: true } })])).rows[0].id;
+      await c.query(`insert into meta.process (page_id, region_id, seq, name, type, point) values ($1, $2, $3, $4, 'grid_dml', 'submit')`, [pageId, grid, rseq, `Save ${label(y.name)}`]);
+      rseq += 10;
+    }
+    return pageNo;
+  });
+}
+
+export async function jsonSourceRoutes(app: FastifyInstance) {
+  const load = async (req: Req, reply: FastifyReply) => {
+    const s = await developer(req, reply);
+    if (!s) return null;
+    const a = await appOr404(req.params.id);
+    if (!a) {
+      reply.code(404).send('Not found');
+      return null;
+    }
+    return { s, a };
+  };
+
+  app.get(`${BASE}/apps/:id(^\\d+$)/json-sources`, async (req: Req, reply) => {
+    const ok = await load(req, reply);
+    if (!ok) return;
+    const { s, a } = ok;
+    const list = listOf(a);
+    const db = await workshopDb(s);
+    const states = await Promise.all(list.map((x) => viewState(db, x).catch(() => new Map<string, boolean>())));
+    const main = html`${appHeader(a, 'shared', s)}
+      <div class="title-row"><h1>JSON sources</h1></div>
+      <p class="muted u-mt0">A JSON source describes a jsonb column: a column per path in its documents, and arrays of objects as rows of their own. pgkiln makes updatable views from it, so forms, reports, interactive grids, REST and MCP read and write the documents like tables. Saving a row changes only the paths it changed: keys the source doesn't know about stay. For a table with its child tables as one document, see SQL Workshop › Table Properties › Document view.</p>
+      ${region('Sources', html`<div class="table-wrap"><table class="report report-reflow"><thead><tr><th scope="col">Name</th><th scope="col">Column</th><th scope="col">Views</th></tr></thead>
+        <tbody>${list.length ? list.map((x, i) => html`<tr><td data-label="Name"><a href="${srcUrl(a.id, x.name)}">${x.name}</a>${x.description ? html` <span class="muted">${x.description}</span>` : ''}</td>
+          <td data-label="Column"><code>${x.schema}.${x.table}.${x.column}</code></td>
+          <td data-label="Views">${[x.view, ...x.arrays.map((y) => y.view)].map((v) => html`<code>${v}</code>${states[i].get(v) ? '' : html` <span class="tag">not made</span>`} `)}</td></tr>`)
+          : html`<tr><td colspan="3" class="empty">No JSON sources yet.</td></tr>`}</tbody></table></div>
+        <p><a class="btn" href="${BASE}/apps/${a.id}/json-sources/source">${icon('plus')} Add JSON source</a></p>`)}`;
+    return send(reply, s, shell(s, 'JSON sources', crumbs(a), main));
+  });
+
+  app.get(`${BASE}/apps/:id(^\\d+$)/json-sources/source`, async (req: Req, reply) => {
+    const ok = await load(req, reply);
+    if (!ok) return;
+    const { s, a } = ok;
+    const name = String((req.query as Record<string, unknown>)?.name ?? '').toUpperCase();
+    const db = await workshopDb(s);
+    const tables = await jsonColumns(db);
+    const x: JsonSource | undefined = name
+      ? listOf(a).find((y) => y.name === name)
+      : { name: '', schema: '', table: '', key: [], column: '', view: '', columns: [], arrays: [] };
+    if (!x) return reply.code(404).send('Not found');
+    const title = name || 'New JSON source';
+    let extra: Raw | '' = '';
+    if (name) {
+      const st = await viewState(db, x).catch(() => new Map<string, boolean>());
+      const made = st.get(x.view) === true;
+      const ddl = jsonSourceDdl(x, { app: a.alias, grants: await tableGrants(db, x.schema, x.table).catch(() => []), replace: st.size > 0 });
+      extra = html`${region('Views', html`
+        <p>${made ? html`${icon('check')} The views exist. Make them again after changing the columns.` : 'The views are not made yet.'}</p>
+        <details><summary>DDL</summary><pre class="source te-ddl" data-code-view="sql" tabindex="0" aria-label="DDL">${ddl.map((d) => `${d};`).join('\n\n')}</pre></details>
+        <form method="post" action="${BASE}/apps/${a.id}/json-sources/views" class="buttons">${csrf(s)}<input type="hidden" name="name" value="${x.name}">
+          <button class="btn btn-hot" name="mode" value="apply">${icon('check')} ${made ? 'Make the views again' : 'Create views'}</button>
+          <button class="btn" name="mode" value="worksheet">${icon('code')} Open in the SQL Worksheet</button></form>`)}
+        ${made ? region('Create page', html`<form method="post" action="${BASE}/apps/${a.id}/json-sources/page">${csrf(s)}<input type="hidden" name="name" value="${x.name}">
+          <p class="muted u-mt0">A form on <code>${x.view}</code>${x.arrays.length ? html` with ${x.arrays.length === 1 ? 'a detail grid' : 'detail grids'} for ${x.arrays.map((y) => y.name).join(', ')}` : ''}. Open it with the key in the URL (<code>P<i>n</i>_${(x.key[0] ?? '').toUpperCase()}</code>), e.g. from a report's link.</p>
+          <div class="form-grid">${input('page_no', 'Page number', '', { type: 'number', required: true })}${input('title', 'Title', x.name.replace(/_/g, ' ').toLowerCase().replace(/^./, (ch) => ch.toUpperCase()), { required: true })}</div>
+          <div class="buttons"><button class="btn">${icon('plus')} Create page</button></div></form>`) : ''}
+        <form method="post" action="${BASE}/apps/${a.id}/json-sources/source/delete" class="danger-zone">${csrf(s)}<input type="hidden" name="name" value="${x.name}">
+          <button class="btn btn-danger" data-confirm="Delete JSON source ${x.name}? Its views stay in the database (drop them in the SQL Workshop).">Delete JSON source</button></form>`;
+    }
+    const main = html`${appHeader(a, 'shared', s)}<div class="title-row"><h1>${title}</h1></div>${region('JSON source', sourceForm(a, s, x, name, [], tables))}${extra}`;
+    return send(reply, s, shell(s, title, crumbs(a, title), main));
+  });
+
+  app.post(`${BASE}/apps/:id(^\\d+$)/json-sources/source`, async (req: Req, reply) => {
+    const ok = await load(req, reply);
+    if (!ok) return;
+    const { s, a } = ok;
+    const b = (req.body ?? {}) as Body;
+    const original = one(b.original).toUpperCase();
+    const db = await workshopDb(s);
+    const tables = await jsonColumns(db);
+    let x = sourceFrom(b, []);
+    const errors: string[] = [];
+    if (!tables.some((t) => t.schema === x.schema && t.table === x.table && t.column === x.column)) errors.push('Choose a table and its json or jsonb column.');
+    else x.key = await primaryKey(db, x.schema, x.table);
+    if (!x.view && x.table) x.view = `${x.table}_${x.column}`.toLowerCase().replace(/[^a-z0-9_]+/g, '_').slice(0, 63);
+    if (!errors.length && one(b.mode) === 'propose') {
+      const p = proposeProfile(await sampleDocuments(db, x.schema, x.table, x.column), { key: x.key, view: x.view });
+      // keep what the developer named; add what the data has besides
+      const have = new Set(x.columns.map((c) => pathText(c.path)));
+      x = { ...x, columns: [...x.columns, ...p.columns.filter((c) => !have.has(pathText(c.path)) && !x.columns.some((y) => y.name === c.name))] };
+      for (const pa of p.arrays) {
+        const mine = x.arrays.find((y) => pathText(y.path) === pathText(pa.path));
+        if (!mine) x.arrays.push(pa);
+        else if (!mine.columns.length) mine.columns = pa.columns;
+      }
+      const title = original || 'New JSON source';
+      const main = html`${appHeader(a, 'shared', s)}<div class="title-row"><h1>${title}</h1></div>
+        <p class="alert alert-info" role="status">Proposed from up to 200 documents: check the names and types, then save.</p>
+        ${region('JSON source', sourceForm(a, s, x, original, [], tables))}`;
+      return send(reply, s, shell(s, title, crumbs(a, title), main));
+    }
+    const problems = errors.length ? errors : await update(a.id, (list) => {
+      const out = jsonSourceProblems(x);
+      const at = original ? list.findIndex((y) => y.name === original) : -1;
+      if (original && at < 0) out.push(`JSON source ${original} no longer exists.`);
+      if (list.some((y, i) => y.name === x.name && i !== at)) out.push(`A JSON source ${x.name} exists already.`);
+      const views = new Map(list.filter((_, i) => i !== at).flatMap((y) => [y.view, ...y.arrays.map((z) => z.view)].map((v) => [`${y.schema}.${v}`, y.name])));
+      for (const v of [x.view, ...x.arrays.map((z) => z.view)]) if (views.has(`${x.schema}.${v}`)) out.push(`The view ${v} belongs to JSON source ${views.get(`${x.schema}.${v}`)}.`);
+      if (out.length) return out;
+      if (at >= 0) list[at] = x;
+      else list.push(x);
+      return [];
+    });
+    if (problems.length) {
+      const title = original || 'New JSON source';
+      const main = html`${appHeader(a, 'shared', s)}<div class="title-row"><h1>${title}</h1></div>${region('JSON source', sourceForm(a, s, x, original, problems, tables))}`;
+      return send(reply.code(422), s, shell(s, title, crumbs(a, title), main));
+    }
+    flash(s, 'JSON source saved.');
+    return back(reply, s, srcUrl(a.id, x.name));
+  });
+
+  app.post(`${BASE}/apps/:id(^\\d+$)/json-sources/source/delete`, async (req: Req, reply) => {
+    const ok = await load(req, reply);
+    if (!ok) return;
+    const { s, a } = ok;
+    const name = one((req.body as Body)?.name).toUpperCase();
+    await update(a.id, (list) => {
+      const at = list.findIndex((y) => y.name === name);
+      if (at >= 0) list.splice(at, 1);
+      return [];
+    });
+    flash(s, `JSON source ${name} deleted. Its views stay in the database.`);
+    return back(reply, s, `${BASE}/apps/${a.id}/json-sources`);
+  });
+
+  // Create views: in one transaction with the developer's SQL Workshop connection, recorded in the Change Log
+  app.post(`${BASE}/apps/:id(^\\d+$)/json-sources/views`, async (req: Req, reply) => {
+    const ok = await load(req, reply);
+    if (!ok) return;
+    const { s, a } = ok;
+    const b = (req.body ?? {}) as Body;
+    const x = listOf(a).find((y) => y.name === one(b.name).toUpperCase());
+    if (!x) return reply.code(404).send('Not found');
+    const to = srcUrl(a.id, x.name);
+    const db = await workshopDb(s);
+    // the catalog again: the table must still be one the connection may use, with that column and key
+    const tables = await jsonColumns(db);
+    const problems = jsonSourceProblems(x);
+    if (!tables.some((t) => t.schema === x.schema && t.table === x.table && t.column === x.column)) problems.push(`${x.schema}.${x.table}.${x.column} is not a json or jsonb column this connection may use.`);
+    else if ((await primaryKey(db, x.schema, x.table)).join(',') !== x.key.join(',')) problems.push(`The primary key of ${x.schema}.${x.table} changed: save the source again.`);
+    const st = problems.length ? new Map() : await viewState(db, x);
+    for (const [v, ours] of st) if (!ours) problems.push(`${x.schema}.${v} exists and is not a view of a JSON source: choose another name.`);
+    if (problems.length) {
+      flash(s, problems.join(' '), 'error');
+      return back(reply, s, to);
+    }
+    const ddl = jsonSourceDdl(x, { app: a.alias, grants: await tableGrants(db, x.schema, x.table), replace: st.size > 0 });
+    if (one(b.mode) === 'worksheet') {
+      s.state.__SQLDRAFT = `${ddl.join(';\n\n')};`;
+      flash(s, 'Check the statements, then run them as a script.');
+      return back(reply, s, `${BASE}/sql/worksheet`);
+    }
+    const c = await db.pool.connect();
+    let failed = '';
+    try {
+      await c.query('begin');
+      for (const d of ddl) await c.query(d);
+      await c.query('commit');
+    } catch (e) {
+      failed = (e as Error).message;
+      await c.query('rollback').catch(() => {});
+    } finally {
+      c.release(true);
+    }
+    await logActivity({ username: s.username, event: 'sql_command', ip: clientIp(req), detail: ddl.join(';\n').slice(0, 2000) });
+    clearCompletions();
+    if (failed) {
+      flash(s, `Nothing was changed: ${failed}`, 'error');
+      return back(reply, s, to);
+    }
+    await recordChanges({ workspaceId: currentWorkspace(s), author: s.username ?? '?', source: 'table', defaultSchema: x.schema, statements: [joinStatements(ddl)] });
+    flash(s, `The views of ${x.name} are made: ${[x.view, ...x.arrays.map((y) => y.view)].join(', ')}.`);
+    return back(reply, s, to);
+  });
+
+  app.post(`${BASE}/apps/:id(^\\d+$)/json-sources/page`, async (req: Req, reply) => {
+    const ok = await load(req, reply);
+    if (!ok) return;
+    const { s, a } = ok;
+    const b = (req.body ?? {}) as Body;
+    const x = listOf(a).find((y) => y.name === one(b.name).toUpperCase());
+    if (!x) return reply.code(404).send('Not found');
+    const pageNo = Number(one(b.page_no));
+    const title = one(b.title).trim().slice(0, 200) || x.name;
+    if (!Number.isInteger(pageNo) || pageNo < 1 || pageNo > 99999) {
+      flash(s, 'The page number is a whole number from 1.', 'error');
+      return back(reply, s, srcUrl(a.id, x.name));
+    }
+    try {
+      await createSourcePage(a.id, x, pageNo, title);
+    } catch (e) {
+      flash(s, (e as Error).message, 'error');
+      return back(reply, s, srcUrl(a.id, x.name));
+    }
+    flash(s, `Page ${pageNo} made: a form on ${x.view}${x.arrays.length ? ' with detail grids' : ''}.`);
+    return back(reply, s, `${BASE}/apps/${a.id}/pages/${pageNo}`);
+  });
+}
+

package/src/builder/object-actions.ts

@@ -34,6 +34,7 @@ export function objectMenu(schema: string, o: MenuObj): Raw {
       ${table ? item(`${BASE}/sql/load`, 'upload', 'Load data') : ''}
       ${table ? item(`${BASE}/sql/objects/fulltext?${new URLSearchParams({ o: o.ref })}`, 'search', 'Add full-text search…') : ''}
       ${table ? item(`${BASE}/sql/objects/vector?${new URLSearchParams({ o: o.ref })}`, 'search', 'Add vector column…') : ''}
+      ${table ? item(`${BASE}/sql/objects/document?${new URLSearchParams({ o: o.ref })}`, 'file', 'Document view…') : ''}
       ${table ? html`${item(q('index'), 'list', 'Add index')}${item(q('trigger'), 'bolt', 'Add trigger')}${item(q('policy'), 'shield', 'Add policy')}` : ''}
       ${RELATIONS.has(o.type) ? item(`${BASE}/sql/rest/object?${new URLSearchParams({ schema, name: o.name })}`, 'globe', 'REST › Enable / Disable') : ''}
       ${item(q('ddl'), 'code', 'Quick DDL to the worksheet')}

package/src/builder/report-settings.ts

@@ -1,3 +1,4 @@
+import { JSON_TYPES, jsonColumnsText, parseJsonColumns } from '../json-sources.ts';
 import type { FastifyInstance } from 'fastify';
 import { designSql } from './websources.ts';
 import pg from 'pg';
@@ -53,6 +54,7 @@ export function parseLinkItems(text: string | undefined): Record<string, string>
 export const linkItemsText = (items: Record<string, string> | undefined) => (items ? Object.entries(items).map(([k, v]) => `${k}=${v}`).join(', ') : '');
 
 interface ReportConfig {
+  json_columns?: { name: string; column: string; path: string; type: string }[];
   page_size?: number;
   pagination?: 'range';
   keyset?: string[];
@@ -151,6 +153,11 @@ export async function reportSettingsForm(pageId: number, appId: number, r: { id:
           <input id="${id('cache_seconds')}" name="cache_seconds" type="number" min="1" max="86400" value="${cfg.cache?.seconds ?? ''}" placeholder="300">
           <small class="help">A submit of the page empties its cache.</small></div>
       </div></fieldset>
+      <fieldset class="prop-group"><legend>JSON path columns</legend><div class="form-grid">
+        <div class="field" data-wide><label class="label" for="${id('json_columns')}">Columns read from a json or jsonb column</label>
+          <input id="${id('json_columns')}" name="json_columns" value="${jsonColumnsText(cfg.json_columns)}" placeholder="city=profile:address.city; first_skill=profile:skills.0.name; since=profile:since:date" class="code">
+          <small class="help">name=column:path[:type]; separate them with ";". A path is keys separated by dots, a number picks an element of an array. Types: ${JSON_TYPES.join(', ')} (text when left out); a value that doesn't read as the type shows empty. They are columns like the others: search, filters, sorting, headings and downloads. To edit documents, use a JSON source (Shared Components).</small></div>
+      </div></fieldset>
       <fieldset class="prop-group"><legend>Columns</legend>
         ${all.length
           ? html`<div class="table-wrap"><table class="report report-reflow"><thead><tr><th>Column</th><th>Heading</th><th>Format mask</th><th>Display</th><th>Shown</th><th>In PDF</th><th>PDF width (mm)</th></tr></thead><tbody>${columnRows}</tbody></table></div>`
@@ -202,6 +209,8 @@ export function mergeReportSettings(config: ReportConfig, b: Record<string, stri
   const seconds = Math.floor(Number(b.cache_seconds || 300));
   set('cache', scope === 'user' || scope === 'session' || scope === 'all' ? { scope, seconds: Math.min(Math.max(seconds || 300, 1), 86_400) } : undefined);
   set('empty', b.empty?.trim() || undefined);
+  const json = parseJsonColumns(b.json_columns);
+  set('json_columns', json.length ? json : undefined);
   set('searchable', b.searchable === 'true' ? undefined : false);
   set('interactive', b.interactive === 'true' ? undefined : false);
   set('sortable', b.sortable === 'true' ? undefined : false);

package/src/builder/shared.ts

@@ -73,7 +73,7 @@ export function sharedGroups(appId: number): { key: string; title: string; icon:
     { key: 'navigation', title: 'Navigation and Search', icon: 'compass', items: [t('nav_entry', 'Navigation Menu'), settings('ui', 'Navigation Bar List'), page('breadcrumbs', 'Breadcrumbs'), t('page_group', 'Page Groups'), t('search_config', 'Search Configurations')] },
     { key: 'ui', title: 'User Interface', icon: 'grid', items: [page('theme', 'Themes and Theme Roller'), settings('ui', 'User Interface Attributes'), t('template_component', 'Template Components'), page('plugins', 'Plug-ins'), page('component-settings', 'Component Settings'), settings('pwa', 'Progressive Web App')] },
     { key: 'files', title: 'Files and Reports', icon: 'file', items: [page('static-files', 'Static Application Files'), t('report_layout', 'Report Layouts'), t('report_query', 'Report Queries'), t('document_template', 'Document Templates')] },
-    { key: 'data', title: 'Data Sources', icon: 'database', items: [t('rest_source', 'REST Data Sources'), t('web_credential', 'Web Credentials'), t('rest_module', 'RESTful Services'), t('data_load_def', 'Data Load Definitions')] },
+    { key: 'data', title: 'Data Sources', icon: 'database', items: [t('rest_source', 'REST Data Sources'), t('web_credential', 'Web Credentials'), t('rest_module', 'RESTful Services'), t('data_load_def', 'Data Load Definitions'), page('json-sources', 'JSON Sources')] },
     { key: 'workflows', title: 'Workflows and Automations', icon: 'activity', items: [t('workflow_definition', 'Workflows'), t('task_definition', 'Task Definitions'), t('automation', 'Automations')] },
     { key: 'globalization', title: 'Globalization', icon: 'globe', items: [page('globalization', 'Translations and Text Messages'), t('dynamic_translation', 'Dynamic Translations'), settings('globalization', 'Globalization Attributes')] },
     { key: 'ai', title: 'Generative AI', icon: 'lightbulb', items: [page('ai', 'AI Services'), page('ai-agents', 'AI Agents'), page('vector-providers', 'Vector Providers')] },

package/src/builder/table-editor.ts

@@ -144,6 +144,7 @@ function form(s: Session, def: TableDef, opts: { editing: TableDef | null; schem
       <button class="btn" name="mode" value="worksheet">${icon('code')} ${tx('te.open_ws')}</button>
       ${editing ? html`<a class="btn" href="${BASE}/sql/objects/fulltext?${new URLSearchParams({ o: opts.ref })}">${icon('search')} ${tx('fts.add')}…</a>` : ''}
       ${editing ? html`<a class="btn" href="${BASE}/sql/objects/vector?${new URLSearchParams({ o: opts.ref })}">${icon('search')} ${tx('vec.add')}…</a>` : ''}
+      ${editing ? html`<a class="btn" href="${BASE}/sql/objects/document?${new URLSearchParams({ o: opts.ref })}">${icon('file')} ${tx('docv.add')}…</a>` : ''}
       <a class="btn" href="${editing ? `${BASE}/sql/objects?${new URLSearchParams({ schema: def.schema, k: 'table', o: opts.ref })}` : `${BASE}/sql/objects?${new URLSearchParams({ schema: def.schema })}`}">${tx('common.cancel')}</a>
     </div></form>`;
 }

package/src/builder/websources.ts

@@ -1,3 +1,4 @@
+import { withJsonColumns } from '../json-sources.ts';
 import type { FastifyInstance } from 'fastify';
 import { owner } from '../db.ts';
 import { html } from '../html.ts';
@@ -124,10 +125,12 @@ export const REST_SOURCE_SPEC: ComponentSpec = {
  * A region's SQL for the builder (columns, checks): with a REST data source,
  * the source's columns without rows, so nothing is fetched.
  */
-export async function designSql(appId: number, r: { source: string | null; rest_source?: string | null }) {
-  if (!r.rest_source) return r.source;
+export async function designSql(appId: number, r: { source: string | null; rest_source?: string | null; config?: Record<string, any> | null }) {
+  // (142) a report's JSON path columns are columns too
+  const json = (sql: string | null) => (sql && r.config?.json_columns ? withJsonColumns(r, sql) : sql);
+  if (!r.rest_source) return json(r.source);
   const s = await owner.one<{ columns: RestColumn[] }>('select columns from meta.rest_source where app_id = $1 and name = $2', [appId, r.rest_source]);
-  return s ? withRest(rowsSql(s.columns, []), r.source) : r.source;
+  return json(s ? withRest(rowsSql(s.columns, []), r.source) : r.source);
 }
 
 // ---------------------------------------------------------------- extras

package/src/i18n/ar.ts

@@ -475,6 +475,30 @@ export const ar: Record<MessageKey, string> = {
   'op.not_contains': "لا يحتوي على",
   'op.null': "فارغ",
   'op.not_null': "غير فارغ",
+  'op.in': "أحد",
+  'op.not_in': "ليس أيًا من",
+  'op.between': "بين",
+  'op.period': "ضمن",
+  'report.value_to': "إلى",
+  'report.values': "القيم",
+  'report.period': "الفترة",
+  'report.and': "و",
+  'report.filter_help': "أحد: قيم مفصولة بفواصل أو محددة في القائمة. بين: يمكن ترك إحدى القيمتين فارغة. ضمن: فترة لعمود تاريخ.",
+  'report.order_chip': "ترتيب أعمدة خاص",
+  'report.order_reset': "العودة إلى ترتيب أعمدة التقرير",
+  'report.order_help': "اسحب عمودًا أو استخدم الأسهم لتغيير الترتيب.",
+  'report.move_up': "نقل {column} لأعلى",
+  'report.move_down': "نقل {column} لأسفل",
+  'report.moved': "{column}: الموضع {n} من {total}",
+  'period.today': "اليوم",
+  'period.yesterday': "أمس",
+  'period.last_7_days': "آخر 7 أيام",
+  'period.last_30_days': "آخر 30 يومًا",
+  'period.next_7_days': "الأيام السبعة القادمة",
+  'period.this_month': "هذا الشهر",
+  'period.last_month': "الشهر الماضي",
+  'period.this_year': "هذه السنة",
+  'period.last_year': "السنة الماضية",
   'grid.delete_row': "حذف الصف {row}",
   'grid.new': "جديد",
   'grid.row_changed': "الصف {row}: تم تغيير هذا الصف خارج الشبكة؛ أعد تحميل الصفحة.",

package/src/i18n/cs.ts

@@ -475,6 +475,30 @@ export const cs: Record<MessageKey, string> = {
   'op.not_contains': "neobsahuje",
   'op.null': "je prázdné",
   'op.not_null': "není prázdné",
+  'op.in': "je jedno z",
+  'op.not_in': "není žádné z",
+  'op.between': "je mezi",
+  'op.period': "je v",
+  'report.value_to': "Do",
+  'report.values': "Hodnoty",
+  'report.period': "Období",
+  'report.and': "a",
+  'report.filter_help': "Je jedno z: hodnoty oddělené čárkami nebo zaškrtnuté v seznamu. Je mezi: jedna z hodnot může zůstat prázdná. Je v: období sloupce s datem.",
+  'report.order_chip': "Vlastní pořadí sloupců",
+  'report.order_reset': "Zpět na pořadí sloupců sestavy",
+  'report.order_help': "Přetáhněte sloupec nebo použijte šipky ke změně pořadí.",
+  'report.move_up': "Posunout {column} nahoru",
+  'report.move_down': "Posunout {column} dolů",
+  'report.moved': "{column}: pozice {n} z {total}",
+  'period.today': "dnes",
+  'period.yesterday': "včera",
+  'period.last_7_days': "posledních 7 dnech",
+  'period.last_30_days': "posledních 30 dnech",
+  'period.next_7_days': "příštích 7 dnech",
+  'period.this_month': "tomto měsíci",
+  'period.last_month': "minulém měsíci",
+  'period.this_year': "tomto roce",
+  'period.last_year': "minulém roce",
   'grid.delete_row': "Smazat řádek {row}",
   'grid.new': "nový",
   'grid.row_changed': "Řádek {row}: tento řádek byl změněn mimo mřížku; načtěte stránku znovu.",

package/src/i18n/da.ts

@@ -475,6 +475,30 @@ export const da: Record<MessageKey, string> = {
   'op.not_contains': "indeholder ikke",
   'op.null': "er tom",
   'op.not_null': "er ikke tom",
+  'op.in': "er en af",
+  'op.not_in': "er ingen af",
+  'op.between': "er mellem",
+  'op.period': "er i",
+  'report.value_to': "Til",
+  'report.values': "Værdier",
+  'report.period': "Periode",
+  'report.and': "og",
+  'report.filter_help': "Er en af: værdier adskilt af kommaer, eller markeret på listen. Er mellem: den ene værdi må være tom. Er i: en periode for en datokolonne.",
+  'report.order_chip': "Egen kolonnerækkefølge",
+  'report.order_reset': "Tilbage til rapportens kolonnerækkefølge",
+  'report.order_help': "Træk en kolonne, eller brug pilene, for at ændre rækkefølgen.",
+  'report.move_up': "Flyt {column} op",
+  'report.move_down': "Flyt {column} ned",
+  'report.moved': "{column}: position {n} af {total}",
+  'period.today': "i dag",
+  'period.yesterday': "i går",
+  'period.last_7_days': "de seneste 7 dage",
+  'period.last_30_days': "de seneste 30 dage",
+  'period.next_7_days': "de næste 7 dage",
+  'period.this_month': "denne måned",
+  'period.last_month': "sidste måned",
+  'period.this_year': "dette år",
+  'period.last_year': "sidste år",
   'grid.delete_row': "Slet række {row}",
   'grid.new': "ny",
   'grid.row_changed': "Række {row}: rækken er ændret uden for gitteret; genindlæs siden.",

package/src/i18n/de.ts

@@ -472,6 +472,30 @@ export const de: Record<MessageKey, string> = {
   'op.not_contains': 'enthält nicht',
   'op.null': 'ist leer',
   'op.not_null': 'ist nicht leer',
+  'op.in': "ist einer von",
+  'op.not_in': "ist keiner von",
+  'op.between': "liegt zwischen",
+  'op.period': "liegt in",
+  'report.value_to': "Bis",
+  'report.values': "Werte",
+  'report.period': "Zeitraum",
+  'report.and': "und",
+  'report.filter_help': "Ist einer von: Werte durch Kommas getrennt oder in der Liste angehakt. Liegt zwischen: ein Wert darf leer bleiben. Liegt in: ein Zeitraum einer Datumsspalte.",
+  'report.order_chip': "Eigene Spaltenreihenfolge",
+  'report.order_reset': "Zurück zur Spaltenreihenfolge des Berichts",
+  'report.order_help': "Ziehen Sie eine Spalte oder verwenden Sie die Pfeile, um die Reihenfolge zu ändern.",
+  'report.move_up': "{column} nach oben",
+  'report.move_down': "{column} nach unten",
+  'report.moved': "{column}: Position {n} von {total}",
+  'period.today': "heute",
+  'period.yesterday': "gestern",
+  'period.last_7_days': "den letzten 7 Tagen",
+  'period.last_30_days': "den letzten 30 Tagen",
+  'period.next_7_days': "den nächsten 7 Tagen",
+  'period.this_month': "diesem Monat",
+  'period.last_month': "dem letzten Monat",
+  'period.this_year': "diesem Jahr",
+  'period.last_year': "dem letzten Jahr",
   'grid.delete_row': 'Zeile {row} löschen',
   'grid.new': 'neu',
   'grid.row_changed': 'Zeile {row}: Diese Zeile wurde außerhalb des Rasters geändert; laden Sie die Seite neu.',