pgkiln 0.71.0 → 0.72.0

package/db/migrations/139_opentelemetry.sql

@@ -0,0 +1,36 @@
+-- =====================================================================
+-- 139: OpenTelemetry (sprint 139, AP-6; APEX: Workspace Utilities →
+--      OpenTelemetry)
+--
+-- The instance's collector and its choices are instance settings in
+-- meta.setting (otel_*: enabled, endpoint, service name, sample ratio,
+-- user attribute, values, signals, builder requests, browser spans;
+-- metrics_endpoint for GET /metrics) and, as only the owner reads it,
+-- meta.instance_setting (otel_headers_enc: headers encrypted with the
+-- instance key; metrics_token_hash), else the standard OTEL_*
+-- environment variables (src/telemetry/config.ts). No table needed.
+--
+-- meta.workspace.telemetry: what the workspace's applications send:
+--   'instance' (the default: the instance's collector, when there is one),
+--   'off' (nothing, not even to the instance's), 'own' (also to the
+--   workspace's own collector, telemetry_endpoint, an OTLP/HTTP base URL
+--   reached through the workspace's network access list and the
+--   installation's allow-list). telemetry_headers_enc: headers for it
+--   (an API key), encrypted like a web credential's secret, write-only.
+--   telemetry_values: send SQL statements with their literals, bind
+--   values, query strings and tool arguments to the own collector.
+--   Installation data: not in the workspace export.
+-- =====================================================================
+
+alter table meta.workspace
+  add column telemetry text not null default 'instance' constraint workspace_telemetry_ok check (telemetry in ('instance', 'off', 'own')),
+  add column telemetry_endpoint text constraint workspace_telemetry_endpoint_ok
+    check (telemetry_endpoint is null or (telemetry_endpoint ~ '^https?://[^\s#@]+$' and length(telemetry_endpoint) <= 500)),
+  add column telemetry_headers_enc text,
+  add column telemetry_values boolean not null default false,
+  add constraint workspace_telemetry_own check (telemetry <> 'own' or telemetry_endpoint is not null);
+
+comment on column meta.workspace.telemetry is 'OpenTelemetry of the workspace''s applications: instance (the instance''s collector), off, own (also its own collector) (139)';
+comment on column meta.workspace.telemetry_endpoint is 'The workspace''s own OTLP/HTTP collector (base URL; /v1/traces and /v1/logs are appended) (139)';
+comment on column meta.workspace.telemetry_headers_enc is 'Headers for the own collector, encrypted with PGKILN_SECRET_KEY (139)';
+comment on column meta.workspace.telemetry_values is 'The own collector also receives values: SQL literals, bind values, query strings (139)';

package/docs/apex-feature-parity.md

@@ -7,7 +7,7 @@ open item and open an issue or pull request; see [CONTRIBUTING.md](../CONTRIBUTI
 Legend: ✅ available · 🟡 partial (see notes) · ❌ not yet · ➖ not planned (a deliberate choice,
 or better served by the PostgreSQL ecosystem; see the notes and [extensions](guide/15-extensions.md))
 
-Last reviewed: 2026-11-03 against **APEX 26.2** (sprint 138: an application's Utilities tab with a dashboard, database object dependencies, change history per component, Upgrade application, UI defaults (APEX's Attribute Dictionary), application and page checksums and an icon browser (AP-5); sprint 137: Create page as a grid of page types with previews, the page types interactive report, smart filters, search page, data loading, tree, timeline, media list, content row, comments, dashboard, wizard, unified task list and workflow console, a Feature tab with nine features that install once, and Create page as copy with shared components mapped or copied (AP-3); sprint 136: vector providers (an AI service with an embedding model or a SQL function), `meta.embed`, *Add vector column* in Table Properties with an HNSW index and a queue that keeps embeddings current, search types vector and hybrid (reciprocal rank fusion with full-text) and a Retrieve (RAG) tool for AI agents, all as the app's role through RLS, on pgvector (AP-1); sprint 135: full-text search configurations, APEX's Oracle Text search type on PostgreSQL full-text search with stemming, ranking, highlights, *Did you mean* and *Add full-text search* in Table Properties (AP-2); sprint 134: the builder also in Italiano, 中文 (简体), 中文 (繁體), 日本語 and 한국어, APEX's ten builder languages plus Nederlands (SEC-5); sprints 121, 123–125: the builder in Deutsch, English, Español, Français, Nederlands and Português, step by step: the frame, sign-in, Workspace home, App Builder, Page Designer menus, the SQL Workshop's home, SQL Worksheet, Object Browser and Table Properties, Shared Components, application settings, users, identity providers, developers and workspaces (WS-7); sprint 122: *Ask in your own words* sets faceted search and smart filter facets, smart filters read a typed phrase (WS-5), and Sample Maps, Sample Approvals, Sample Data Loading and Sample Files in the App Gallery (WS-8); sprint 120: the App Gallery's single-topic sample apps (charts, cards, calendar, interactive grids, dynamic actions, reporting), the UX Pattern Catalog and a Utility Apps tab with Lookup Lists, Release Notes and Announcements (WS-8); sprint 119: data security per table, a roles × columns matrix and masked values per application role in a masking view, the same identity for pages, AI tools, REST, AutoREST and MCP (WS-13); sprint 118: Administration → Mail queue with delivery per hour and per workspace, the slowest provider calls and failures with Retry and Purge, a workspace's own mail service, and workspace network access lists that narrow the installation's allow-list (WS-9); sprint 117: Pillar Light and Pillar Dark navigation appearances with a header strip, clearer navigation tree states, subtle colour utilities, RTL; sprint 116: the comparison moved from 26.1 to 26.2, with a section for the 26.2 workspace items WS-1 … WS-13 and OCI managed identities; the App Builder list with Updated by, working copies and the lock; `pgkiln import --only`, a file-level import; stale rows on facet menus and blueprints fixed; sprint 115: popup LOVs that load on scroll, Wait Popup dynamic actions and dynamic actions on menu entries; sprint 114: Workflow 26.2 with custom task outcomes, task events and notifications, Send E-Mail and draft steps, migrating running instances and the diagram as SVG / PNG; sprint 113: Team Development with issues, labels, milestones, templates, a board and a set-up wizard; sprint 112: AI interactive reports with chips, Undo, computed columns and a phrase parser that needs no AI service; sprint 111: AI Agents with tools as a shared component, a Navigate tool with checksummed URLs, Gemini and OpenAI-compatible services; sprint 110: Data Reporter as a workspace tool with datasets, reporting applications, editors and viewers, published reports with stable URLs and pins; sprint 109: the Workspace home above the App Builder and SQL Workshop, with a summary, top applications, users and developers, and an optional release check; sprint 108: the code editor's syntax colours in the builder's light and dark themes, read-only SQL highlighted, the Data Modeler in dark mode; sprint 107: a transaction mode in the SQL Worksheet with Commit and Rollback; DA-18: JavaScript (MLE) not planned, plv8 where installed; sprint 106: the change log of DDL change sets with Flyway, Liquibase and SQL downloads; sprint 104: RESTful Services modules, templates and handlers with PL/pgSQL and media handlers, privileges, OpenAPI and Try it; sprint 103: Schema Export / Import with pg_dump and pg_restore; sprint 102: the SQL Monitor with live sessions, Top SQL and a statement's plan; sprint 101: Scheduling of pg_cron jobs with a schedule builder, run now and run history; sprint 100: RESTful Services, tables and views published as REST endpoints with paging, filters, keys and OpenAPI; sprint 99: Administration → Database Users for PostgreSQL roles; sprint 98: the Data Modeler with relational diagrams, DDL and a data dictionary; sprint 97: charts of a query in the worksheet, saved charts and their dashboard; sprint 96: the SQL Workshop's Launchpad with pins and tool groups, the builder's search with Ctrl+K; sprint 95: Table Properties: create, edit or copy a table in a form with a DDL preview and Apply in one transaction; sprint 94: an object's menu with Open in a panel, Quick DDL, Add index / trigger / policy and Drop with dependents; sprint 93: worksheets saved as SQL Scripts with tabs and a Files list, Format, Convert case, find and replace, bind and substitution variables; sprint 92: the worksheet's Query Result with sort, value dialog, copy and downloads, plans as a tree with the slowest step and misestimates, SQL History with status, search and run again; sprint 91: the SQL Worksheet like Database Actions' SQL: navigator, Run Statement at the cursor and Run Script with a log per statement, Explain Plan, Autotrace, notices, history, Run as an application role; sprint 90: Comment Out in the page designer, gallery labels in title case; sprint 89: the Object Browser's schema, one row of tabs, blank NULLs, Modify column; sprint 88: home cards with colour bands and round icons, the hero on the page background, a page footer with the version and Back to top; sprint 87: the Create Application wizard reloads the page list for a schema, RLS badges, the alias from the name; sprint 86: the page designer by page number, HTML error pages in the builder; sprint 85: the Form Designer, beyond APEX; sprint 84: the date picker, facet menus; sprint 83: Component Settings and Component Groups; sprint 82: Report Queries; sprint 81: Application Computations and Shortcuts in Shared Components; sprint 80: the master grid's first row selected when the page opens and after a search; grid cells as text with their format mask until edited, with cell navigation by arrow keys, Enter, F2 and Escape; sprint 79: generated dashboards with sums, donuts and lines, leaving out charts that say nothing; applications open in their own light or dark mode until the user chooses, per application; a user menu with Appearance and Sign in when signed out; the developer toolbar's App and Page texts and debug state; the report search's column scope in the magnifier; sprint 78: generated faceted search reports with one column per foreign key, the faceted search's total row count, Show all and Yes/No values, one search field, friendly headings and number formats in generated grids and master-detail pages, friendly names in the Create Application wizard's page list; sprint 77: the Page Designer's layout and tree by display position, button positions, the tree's context menu with Duplicate and Copy to other page, Show Common / Show All, Go to group and the Template Options dialog; display positions, button positions and side-column page templates at runtime; sprint 76: the interactive report's menus and pagination, date items in the date format; sprint 75: the SQL Workshop's Object Browser and tabs; sprint 74: the builder's information architecture: Shared Components grid, Settings tabs, application home search and tasks; sprint 73: the Create Application wizard on one page with page types and features; sprint 72: the runtime chrome compared with APEX 26.2's: the developer toolbar, the Iris look, the navigation bar's Help menu and Install button; sprint 71: generated applications compared with APEX 26.2's: labels, foreign key names in reports, file columns, format masks, the Home page and the dashboard; sprint 70: treemap, sunburst and timeline charts; sprint 69: dragging tasks on a Gantt chart; sprint 68: the Data Reporter's welcome page, Find my workspace; sprint 67: REST source catalogs from OpenAPI documents, a workspace's export and import (two rows added); sprint 66: a blueprint from an existing application; sprint 65: facet charts; sprint 64: approval vacation rules and expiration policies; sprint 63: range, box plot and stock charts; sprint 62: workspace preferences (SQL Workshop and RESTful services per workspace), extension links, Manage backups across applications, SQL Workshop utilities with object reports, About Database and a database monitor; sprint 61: the builder's screens compared with APEX 26.1: the workspace sign-in page, the Administration menu with workspace message, environment banner and utilization, the SQL Workshop home, the App Gallery, application groups and cross application reports (four rows added); sprint 60: isolated workspaces with their own database logins, row-level security on pgkiln's tables and workspace database roles, workspace administrators; sprint 59: the Page Designer's help pane, Quick edit from the developer toolbar; sprint 58: page groups; sprint 57: region templates hero, title bar, buttons container, inline dialog and drawer, wizard with wizard progress, the carousel; sprint 56: Generate DDL and Schema Comparison in the SQL Workshop, `pgkiln ddl` and `pgkiln compare`; sprint 55: an editable login page with a Login region and the Minimal page template, Forgot password through the application's mail service; sprint 54: interactive grid views, the single row view, group by and chart; sprint 53: application settings, user preferences and items kept per user, dynamic translations, `meta.data_export` and data export downloads, display image and geocoded address items; sprint 52: every dynamic action event and action, Ajax callbacks, an error handling function, the session timeout warning; sprint 51: checked imports, application backups, availability and Run only, feedback, type-specific attribute help; sprint 50: lost update detection, warn on unsaved changes, the URL and search regions with search configurations, the developer toolbar with a session state viewer, and e-mail through SMTP with OAuth2, Microsoft Graph, Amazon SES and HTTP APIs; import checks, application availability, property help and e-mail compared again; 2026-10-06: pgkiln 0.31.0, in progress: static application files and the Execute JavaScript dynamic action; region, item, dynamic action and process plug-ins; conditional and dynamic theme styles; XML and Excel in `meta.parse_data`, zips in SQL, `meta.v_boolean`; file items in S3-compatible object storage; map layers by the visible area and as vector tiles; a graphical query builder; tenants for workflows and tasks; application files as YAML and region static IDs; Lucide icons and icon modifiers; PWA push notifications; 0.30.0: 136 icons and an icon picker; an axe-core accessibility audit; cropping pictures; session sharing between applications; twenty-two built-in languages; 0.29.0: workspaces; the Iris base style; drawers; built-in template components including the metric card; Theme Roller dark-mode colours, live preview and template options on items and report columns; report selection across pages; instance settings; fourteen built-in languages; 0.28.0: AI with Claude and OpenAI; 0.27.0: Data Reporter; sample data generator; create application from several sheets, pasted data or existing tables; 0.26.0: working copies with a three-way merge; theme, library and boilerplate applications with subscriptions; 0.25.0: Gantt, pyramid and polar charts; map layers, clustering and server-side spatial filters; REST data sources that write back from forms and grids and synchronise into tables, OAuth2 password and refresh-token grants; debug messages per request and an install/upgrade log; `meta.web_request()` and `meta.parse_data()`; Theme Roller style variants and template options; 0.24.0: automations with several actions, error handling per row and runs from SQL; invoke-API workflow steps; Unload Data in the Data Workshop; create page wizards for cards, calendar, chart, map, faceted search, form and master-detail; create an application from a file; 0.23.0: interactive grid aggregates, frozen/moved/resized columns, layouts and saved grid reports, master-detail, row actions and copy/paste; download processes, execution chains in the background, workflow processes, branches to a function's URL or another app, the Dialog Closed event; number format masks, automatic time zone, runtime messages in German, French and Spanish; SQL scripts, Quick SQL, a query builder, XML loading and data load definitions; custom authentication, lists, page and application locks with developer comments, supporting objects; 0.22.0: keyset paging, report PDFs and REST collections streamed from a cursor, database-account authentication; 0.21.0: HTTP-header authentication behind a reverse proxy; 0.20.0: a popup list of values with server-side search; 0.19.0: row-range pagination, maximum row counts and row limits, lazy loading and caching of regions, streamed CSV/Excel downloads; 0.18.0: calendar week/day/list views with drag and drop, bubble/gauge/funnel/radar charts and drill-down, smart filters, a region display selector and more facet types, computations, conditional branches, build options, menu buttons, REST data sources and web credentials, rich text/Markdown, rating, combobox, date range and QR code items; 0.17.0: an App Builder home, workspace dashboard and utilities like APEX's; 0.16.0: a Page Designer with drag-and-drop layout, a code editor with autocomplete, template components and plug-ins, parallel branches and versions of workflows, the `pgkiln` command line with one file per component; 0.15.0: drop and paste files, heat maps, filtering a report by the map area; 0.14.0: stacked, combo, scatter and pie charts, several files per upload item; 0.13.0: map and tree regions; 0.12.0: REST modules in the builder; 0.11.0: workflows and Progressive Web Apps; 0.10.0: LDAP, SAML, "Keep me signed in", document templates, JSON loading, approvals and the task list; pgkiln installs no application, HR is an example).
+Last reviewed: 2026-11-04 against **APEX 26.2** (sprint 139: OpenTelemetry, traces with a span per SQL statement, web call, AI request and tool call, logs, metrics and `/metrics` for Prometheus, browser spans relayed by pgkiln, per workspace off or its own collector (AP-6); sprint 138: an application's Utilities tab with a dashboard, database object dependencies, change history per component, Upgrade application, UI defaults (APEX's Attribute Dictionary), application and page checksums and an icon browser (AP-5); sprint 137: Create page as a grid of page types with previews, the page types interactive report, smart filters, search page, data loading, tree, timeline, media list, content row, comments, dashboard, wizard, unified task list and workflow console, a Feature tab with nine features that install once, and Create page as copy with shared components mapped or copied (AP-3); sprint 136: vector providers (an AI service with an embedding model or a SQL function), `meta.embed`, *Add vector column* in Table Properties with an HNSW index and a queue that keeps embeddings current, search types vector and hybrid (reciprocal rank fusion with full-text) and a Retrieve (RAG) tool for AI agents, all as the app's role through RLS, on pgvector (AP-1); sprint 135: full-text search configurations, APEX's Oracle Text search type on PostgreSQL full-text search with stemming, ranking, highlights, *Did you mean* and *Add full-text search* in Table Properties (AP-2); sprint 134: the builder also in Italiano, 中文 (简体), 中文 (繁體), 日本語 and 한국어, APEX's ten builder languages plus Nederlands (SEC-5); sprints 121, 123–125: the builder in Deutsch, English, Español, Français, Nederlands and Português, step by step: the frame, sign-in, Workspace home, App Builder, Page Designer menus, the SQL Workshop's home, SQL Worksheet, Object Browser and Table Properties, Shared Components, application settings, users, identity providers, developers and workspaces (WS-7); sprint 122: *Ask in your own words* sets faceted search and smart filter facets, smart filters read a typed phrase (WS-5), and Sample Maps, Sample Approvals, Sample Data Loading and Sample Files in the App Gallery (WS-8); sprint 120: the App Gallery's single-topic sample apps (charts, cards, calendar, interactive grids, dynamic actions, reporting), the UX Pattern Catalog and a Utility Apps tab with Lookup Lists, Release Notes and Announcements (WS-8); sprint 119: data security per table, a roles × columns matrix and masked values per application role in a masking view, the same identity for pages, AI tools, REST, AutoREST and MCP (WS-13); sprint 118: Administration → Mail queue with delivery per hour and per workspace, the slowest provider calls and failures with Retry and Purge, a workspace's own mail service, and workspace network access lists that narrow the installation's allow-list (WS-9); sprint 117: Pillar Light and Pillar Dark navigation appearances with a header strip, clearer navigation tree states, subtle colour utilities, RTL; sprint 116: the comparison moved from 26.1 to 26.2, with a section for the 26.2 workspace items WS-1 … WS-13 and OCI managed identities; the App Builder list with Updated by, working copies and the lock; `pgkiln import --only`, a file-level import; stale rows on facet menus and blueprints fixed; sprint 115: popup LOVs that load on scroll, Wait Popup dynamic actions and dynamic actions on menu entries; sprint 114: Workflow 26.2 with custom task outcomes, task events and notifications, Send E-Mail and draft steps, migrating running instances and the diagram as SVG / PNG; sprint 113: Team Development with issues, labels, milestones, templates, a board and a set-up wizard; sprint 112: AI interactive reports with chips, Undo, computed columns and a phrase parser that needs no AI service; sprint 111: AI Agents with tools as a shared component, a Navigate tool with checksummed URLs, Gemini and OpenAI-compatible services; sprint 110: Data Reporter as a workspace tool with datasets, reporting applications, editors and viewers, published reports with stable URLs and pins; sprint 109: the Workspace home above the App Builder and SQL Workshop, with a summary, top applications, users and developers, and an optional release check; sprint 108: the code editor's syntax colours in the builder's light and dark themes, read-only SQL highlighted, the Data Modeler in dark mode; sprint 107: a transaction mode in the SQL Worksheet with Commit and Rollback; DA-18: JavaScript (MLE) not planned, plv8 where installed; sprint 106: the change log of DDL change sets with Flyway, Liquibase and SQL downloads; sprint 104: RESTful Services modules, templates and handlers with PL/pgSQL and media handlers, privileges, OpenAPI and Try it; sprint 103: Schema Export / Import with pg_dump and pg_restore; sprint 102: the SQL Monitor with live sessions, Top SQL and a statement's plan; sprint 101: Scheduling of pg_cron jobs with a schedule builder, run now and run history; sprint 100: RESTful Services, tables and views published as REST endpoints with paging, filters, keys and OpenAPI; sprint 99: Administration → Database Users for PostgreSQL roles; sprint 98: the Data Modeler with relational diagrams, DDL and a data dictionary; sprint 97: charts of a query in the worksheet, saved charts and their dashboard; sprint 96: the SQL Workshop's Launchpad with pins and tool groups, the builder's search with Ctrl+K; sprint 95: Table Properties: create, edit or copy a table in a form with a DDL preview and Apply in one transaction; sprint 94: an object's menu with Open in a panel, Quick DDL, Add index / trigger / policy and Drop with dependents; sprint 93: worksheets saved as SQL Scripts with tabs and a Files list, Format, Convert case, find and replace, bind and substitution variables; sprint 92: the worksheet's Query Result with sort, value dialog, copy and downloads, plans as a tree with the slowest step and misestimates, SQL History with status, search and run again; sprint 91: the SQL Worksheet like Database Actions' SQL: navigator, Run Statement at the cursor and Run Script with a log per statement, Explain Plan, Autotrace, notices, history, Run as an application role; sprint 90: Comment Out in the page designer, gallery labels in title case; sprint 89: the Object Browser's schema, one row of tabs, blank NULLs, Modify column; sprint 88: home cards with colour bands and round icons, the hero on the page background, a page footer with the version and Back to top; sprint 87: the Create Application wizard reloads the page list for a schema, RLS badges, the alias from the name; sprint 86: the page designer by page number, HTML error pages in the builder; sprint 85: the Form Designer, beyond APEX; sprint 84: the date picker, facet menus; sprint 83: Component Settings and Component Groups; sprint 82: Report Queries; sprint 81: Application Computations and Shortcuts in Shared Components; sprint 80: the master grid's first row selected when the page opens and after a search; grid cells as text with their format mask until edited, with cell navigation by arrow keys, Enter, F2 and Escape; sprint 79: generated dashboards with sums, donuts and lines, leaving out charts that say nothing; applications open in their own light or dark mode until the user chooses, per application; a user menu with Appearance and Sign in when signed out; the developer toolbar's App and Page texts and debug state; the report search's column scope in the magnifier; sprint 78: generated faceted search reports with one column per foreign key, the faceted search's total row count, Show all and Yes/No values, one search field, friendly headings and number formats in generated grids and master-detail pages, friendly names in the Create Application wizard's page list; sprint 77: the Page Designer's layout and tree by display position, button positions, the tree's context menu with Duplicate and Copy to other page, Show Common / Show All, Go to group and the Template Options dialog; display positions, button positions and side-column page templates at runtime; sprint 76: the interactive report's menus and pagination, date items in the date format; sprint 75: the SQL Workshop's Object Browser and tabs; sprint 74: the builder's information architecture: Shared Components grid, Settings tabs, application home search and tasks; sprint 73: the Create Application wizard on one page with page types and features; sprint 72: the runtime chrome compared with APEX 26.2's: the developer toolbar, the Iris look, the navigation bar's Help menu and Install button; sprint 71: generated applications compared with APEX 26.2's: labels, foreign key names in reports, file columns, format masks, the Home page and the dashboard; sprint 70: treemap, sunburst and timeline charts; sprint 69: dragging tasks on a Gantt chart; sprint 68: the Data Reporter's welcome page, Find my workspace; sprint 67: REST source catalogs from OpenAPI documents, a workspace's export and import (two rows added); sprint 66: a blueprint from an existing application; sprint 65: facet charts; sprint 64: approval vacation rules and expiration policies; sprint 63: range, box plot and stock charts; sprint 62: workspace preferences (SQL Workshop and RESTful services per workspace), extension links, Manage backups across applications, SQL Workshop utilities with object reports, About Database and a database monitor; sprint 61: the builder's screens compared with APEX 26.1: the workspace sign-in page, the Administration menu with workspace message, environment banner and utilization, the SQL Workshop home, the App Gallery, application groups and cross application reports (four rows added); sprint 60: isolated workspaces with their own database logins, row-level security on pgkiln's tables and workspace database roles, workspace administrators; sprint 59: the Page Designer's help pane, Quick edit from the developer toolbar; sprint 58: page groups; sprint 57: region templates hero, title bar, buttons container, inline dialog and drawer, wizard with wizard progress, the carousel; sprint 56: Generate DDL and Schema Comparison in the SQL Workshop, `pgkiln ddl` and `pgkiln compare`; sprint 55: an editable login page with a Login region and the Minimal page template, Forgot password through the application's mail service; sprint 54: interactive grid views, the single row view, group by and chart; sprint 53: application settings, user preferences and items kept per user, dynamic translations, `meta.data_export` and data export downloads, display image and geocoded address items; sprint 52: every dynamic action event and action, Ajax callbacks, an error handling function, the session timeout warning; sprint 51: checked imports, application backups, availability and Run only, feedback, type-specific attribute help; sprint 50: lost update detection, warn on unsaved changes, the URL and search regions with search configurations, the developer toolbar with a session state viewer, and e-mail through SMTP with OAuth2, Microsoft Graph, Amazon SES and HTTP APIs; import checks, application availability, property help and e-mail compared again; 2026-10-06: pgkiln 0.31.0, in progress: static application files and the Execute JavaScript dynamic action; region, item, dynamic action and process plug-ins; conditional and dynamic theme styles; XML and Excel in `meta.parse_data`, zips in SQL, `meta.v_boolean`; file items in S3-compatible object storage; map layers by the visible area and as vector tiles; a graphical query builder; tenants for workflows and tasks; application files as YAML and region static IDs; Lucide icons and icon modifiers; PWA push notifications; 0.30.0: 136 icons and an icon picker; an axe-core accessibility audit; cropping pictures; session sharing between applications; twenty-two built-in languages; 0.29.0: workspaces; the Iris base style; drawers; built-in template components including the metric card; Theme Roller dark-mode colours, live preview and template options on items and report columns; report selection across pages; instance settings; fourteen built-in languages; 0.28.0: AI with Claude and OpenAI; 0.27.0: Data Reporter; sample data generator; create application from several sheets, pasted data or existing tables; 0.26.0: working copies with a three-way merge; theme, library and boilerplate applications with subscriptions; 0.25.0: Gantt, pyramid and polar charts; map layers, clustering and server-side spatial filters; REST data sources that write back from forms and grids and synchronise into tables, OAuth2 password and refresh-token grants; debug messages per request and an install/upgrade log; `meta.web_request()` and `meta.parse_data()`; Theme Roller style variants and template options; 0.24.0: automations with several actions, error handling per row and runs from SQL; invoke-API workflow steps; Unload Data in the Data Workshop; create page wizards for cards, calendar, chart, map, faceted search, form and master-detail; create an application from a file; 0.23.0: interactive grid aggregates, frozen/moved/resized columns, layouts and saved grid reports, master-detail, row actions and copy/paste; download processes, execution chains in the background, workflow processes, branches to a function's URL or another app, the Dialog Closed event; number format masks, automatic time zone, runtime messages in German, French and Spanish; SQL scripts, Quick SQL, a query builder, XML loading and data load definitions; custom authentication, lists, page and application locks with developer comments, supporting objects; 0.22.0: keyset paging, report PDFs and REST collections streamed from a cursor, database-account authentication; 0.21.0: HTTP-header authentication behind a reverse proxy; 0.20.0: a popup list of values with server-side search; 0.19.0: row-range pagination, maximum row counts and row limits, lazy loading and caching of regions, streamed CSV/Excel downloads; 0.18.0: calendar week/day/list views with drag and drop, bubble/gauge/funnel/radar charts and drill-down, smart filters, a region display selector and more facet types, computations, conditional branches, build options, menu buttons, REST data sources and web credentials, rich text/Markdown, rating, combobox, date range and QR code items; 0.17.0: an App Builder home, workspace dashboard and utilities like APEX's; 0.16.0: a Page Designer with drag-and-drop layout, a code editor with autocomplete, template components and plug-ins, parallel branches and versions of workflows, the `pgkiln` command line with one file per component; 0.15.0: drop and paste files, heat maps, filtering a report by the map area; 0.14.0: stacked, combo, scatter and pie charts, several files per upload item; 0.13.0: map and tree regions; 0.12.0: REST modules in the builder; 0.11.0: workflows and Progressive Web Apps; 0.10.0: LDAP, SAML, "Keep me signed in", document templates, JSON loading, approvals and the task list; pgkiln installs no application, HR is an example).
 
 ## At a glance
 
@@ -22,9 +22,9 @@ Last reviewed: 2026-11-03 against **APEX 26.2** (sprint 138: an application's Ut
 | Globalization | 6 | 0 | 0 | 0 | One translated app like 26.1 with dynamic translations of table values; number format masks and automatic time zone; twenty-two built-in languages |
 | Data and integration | 13 | 0 | 0 | 2 | REST source catalogs from OpenAPI documents; Generate DDL and schema comparison; `meta.data_export` and data export downloads (CSV, JSON, XML, HTML, Excel, PDF); e-mail through SMTP (password or OAuth2), Microsoft Graph, Amazon SES and HTTP APIs; REST APIs via PostgREST, REST data sources that write back and synchronise, web credentials with OAuth2 grants, CSV/XLSX/JSON/XML loading with saved definitions and unloading, SQL scripts and Quick SQL, report PDFs and document templates |
 | Workflow, automation and AI | 6 | 0 | 0 | 0 | Scheduled automations with several actions and runs from SQL, approvals, a task list and workflows with parallel branches, versions and invoke-API steps; AI with Claude or OpenAI: *Generate text with AI*, an assistant region with tools, natural-language report filters and blueprints |
-| Administration | 14 | 0 | 0 | 0 | An APEX-like sign-in with a workspace and a Workspace home; a mail queue dashboard, a workspace's own mail service and network access list (26.1 / 26.2); a workspace's components exported and imported as one file; workspace message, environment banner and utilization; isolated workspaces with their own database logins and roles, and workspace administrators; application availability and Run only, automatic backups, user feedback; a developer toolbar with a session state viewer and debug per session; Top SQL per app; debug messages per request; an install/upgrade log |
+| Administration | 15 | 0 | 0 | 0 | OpenTelemetry traces, logs and metrics to the team's own monitoring and `/metrics` for Prometheus (24.2); An APEX-like sign-in with a workspace and a Workspace home; a mail queue dashboard, a workspace's own mail service and network access list (26.1 / 26.2); a workspace's components exported and imported as one file; workspace message, environment banner and utilization; isolated workspaces with their own database logins and roles, and workspace administrators; application availability and Run only, automatic backups, user feedback; a developer toolbar with a session state viewer and debug per session; Top SQL per app; debug messages per request; an install/upgrade log |
 | APEX 26.2 workspace (WS-1 … WS-13) | 13 | 0 | 0 | 1 | Workspace home, Data Reporter as a workspace tool, Team Development, AI Agents, AI interactive reports, Workflow 26.2, the mail queue, workspace mail and network access, the 26.1 component details, the App Builder list's details, the Pillar navigation appearances and data security per table (rows, columns and masked values), the App Gallery's single-topic sample apps, UX Pattern Catalog and Utility Apps; the builder in eleven languages (APEX's ten and Nederlands) |
-| **Total** | **162** | **0** | **0** | **4** | 166 APEX features compared: 98% available, none missing, 4 deliberately not planned |
+| **Total** | **163** | **0** | **0** | **4** | 167 APEX features compared: 98% available, none missing, 4 deliberately not planned |
 
 (Counts are of the rows in the tables below. A review on 2026-10-06 added 18 rows for APEX features that were
 not compared before, one on 2026-10-12 added two more (property help, application availability), and one on
@@ -33,7 +33,7 @@ application reports, Manage Service), so the share available dropped without any
 two, REST source catalogs and the workspace export, with their features. Sprint 116 compared with APEX 26.2 and added
 the section [APEX 26.2 workspace](#apex-262-workspace-ws-1--ws-13): fourteen rows, the 26.2 workspace items WS-1 … WS-13
 and OCI managed identities. Some of them repeat a row above from the workspace's side; they are counted on purpose, as
-the 26.2 comparison.)
+the 26.2 comparison. Sprint 139 added OpenTelemetry (24.2), found missing by the 27 Oct pass (AP-6).)
 
 ## App Builder and development
 
@@ -233,6 +233,7 @@ the 26.2 comparison.)
 | Automatic application backups, page groups | ✅ | (sprint 51) **Backups**: before every import that replaces an application (CLI, MCP, working copy merge, restore), daily when it changed, and by hand; the newest 25 (configurable) are kept; download and restore from the builder ([chapter 3](guide/03-builder.md#backups)); (sprint 62) Workspace utilities → **Manage backups** across the workspace's applications, with a backup of all of them at once. (sprint 58) **Page groups**: Shared Components → Page groups, a page's *Page group*, the page list by group with a filter ([chapter 3](guide/03-builder.md#page-groups)) |
 | Application availability (Available, Developers only, Restricted access, Unavailable; build status *Run only*) | ✅ | (sprint 51) Available, available to developers only (a builder session in the same browser), restricted to named users, or unavailable with a message, for pages, file downloads, REST modules and PostgREST; build status **Run only** refuses builder changes as if the application were locked, while `pgkiln import --replace` still deploys. Installation settings: not exported, kept by an import ([chapter 3](guide/03-builder.md#availability-and-build-status)) |
 | Monitoring (top SQL) | ✅ | Activity per app; Top SQL per app's database role from `pg_stat_statements` (sortable, resettable) |
+| OpenTelemetry (Workspace Utilities → OpenTelemetry, 24.2: client logging and token relay to an OTLP collector) | ✅ | (sprint 139, AP-6) Workspace utilities → **OpenTelemetry** ([chapter 1](guide/01-installation.md#monitoring-with-opentelemetry)) or the standard `OTEL_*` variables: OTLP/HTTP to any Collector. A trace per request with a span per SQL statement, REST data source and web call (W3C `traceparent` passed on), AI request (tokens) and tool call; mail and workflow steps as traces of their own; head sampling that keeps every failed trace; values (SQL literals, binds, query strings) only when chosen, the user hashed. Logs of errors, refused access and failed sign-ins; metrics of request durations, errors, AI tokens and the queues; `GET /metrics` for Prometheus. Browser spans of the page load and Ajax calls relayed by pgkiln (APEX's token relay: no collector token reaches the browser). Per workspace: off, or also its own collector through its network access list. Beyond APEX: SQL spans and Prometheus |
 | Workspace export and import (Workspace Utilities → Export) | ✅ | (sprint 67) Workspace utilities → **Export and import** (administrators and the workspace's administrators): settings (message, banner, preferences), developers and workspace administrators, application groups, extension links, REST source catalogs, SQL scripts, blueprints and data generators as one file; imported into the current workspace in one transaction, keeping or replacing components with the same name and never deleting ([chapter 3](guide/03-builder.md#export-and-import)). Applications are exported on their own; isolation, database roles and passwords never leave the installation |
 | Workspaces (multi-tenant) | ✅ | Workspaces group applications and developers: the builder shows and opens only the applications of a developer's workspaces (administrators: all), a current workspace with a switcher, new and imported applications go into it, administrators add workspaces, choose their developers and move applications ([chapter 3](guide/03-builder.md#workspaces)). (sprint 60) **Isolated workspaces** are a tenant boundary: the applications run on a PostgreSQL login of the workspace's own and its developers' SQL Workshop on another, with row-level security on pgkiln's tables, so SQL that resets its role can't reach other workspaces' applications, data, sessions or accounts; **workspace database roles** (APEX: workspace schemas) assigned by administrators; **workspace administrators** who manage their workspace's developers; SQL scripts, blueprints and sample data per workspace ([isolated workspaces](guide/03-builder.md#isolated-workspaces)). **Missing:** user directories per workspace (accounts are installation-wide, visible to the workspaces whose applications they use), developers created by a workspace administrator |
 | Mail queue reporting: delivery health, performance, failures and usage per workspace (26.2) | ✅ | (sprint 118, WS-9) Administration → **Mail queue** ([chapter 3](guide/03-builder.md#mail-queue)): queued, sent, failed and retrying mail per hour and per workspace / application, the share delivered, the average and the 10 slowest provider calls, and every failure with the provider's answer (an SMTP reply such as `550 5.1.1`), with **Retry** and **Purge** (one or all failed). Administrators see every workspace, a workspace's administrators their own |

package/docs/guide/01-installation.md

@@ -197,6 +197,15 @@ which is read at startup; real environment variables take precedence.
 | `LOG_LEVEL` | `info` | `fatal`, `error`, `warn`, `info`, `debug`, `trace` |
 | `PGKILN_RELEASE_CHECK` | `off` | `on`: the server asks GitHub once a day for a newer pgkiln release and the [Workspace home](03-builder.md#the-release-check) announces it. Administrators can also set it in the builder (Instance settings → *New versions*), which wins over the variable. Keep it off where the server may not reach the internet |
 | `PGKILN_RELEASE_URL` | GitHub's releases API | For tests and mirrors only: the address of a JSON document like GitHub's `releases/latest` |
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | *(none)* | (0.72) The OpenTelemetry Collector's OTLP/HTTP address (e.g. `http://otel-collector:4318`): pgkiln sends [traces, logs and metrics](#monitoring-with-opentelemetry) there. Workspace utilities → **OpenTelemetry** wins over it and over the other `OTEL_*` variables below |
+| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, `…_LOGS_ENDPOINT`, `…_METRICS_ENDPOINT` | the endpoint + `/v1/traces` … | A full URL per signal (only when the builder sets no collector URL) |
+| `OTEL_EXPORTER_OTLP_HEADERS` | *(none)* | Headers sent to the collector, `name=value,name2=value2` (an API key) |
+| `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES` | `pgkiln`, *(none)* | The `service.name` and other resource attributes (`deployment.environment=prod`) |
+| `OTEL_TRACES_SAMPLER`, `OTEL_TRACES_SAMPLER_ARG` | always on | `traceidratio` with a ratio from 0 to 1 sends that share of traces; a trace with an error is always sent |
+| `OTEL_TRACES_EXPORTER`, `OTEL_LOGS_EXPORTER`, `OTEL_METRICS_EXPORTER` | `otlp` | `none` leaves that signal out |
+| `OTEL_METRIC_EXPORT_INTERVAL` | `60000` | Milliseconds between metric exports (5 s to 1 h) |
+| `OTEL_SDK_DISABLED` | `false` | `true` turns the instance's export off, whatever the builder says |
+| `PGKILN_METRICS`, `PGKILN_METRICS_TOKEN` | `off`, *(none)* | `on` opens `GET /metrics` (Prometheus) for administrators and for scrapers that send `Authorization: Bearer <PGKILN_METRICS_TOKEN>` |
 
 ## npm scripts
 
@@ -355,6 +364,105 @@ run several instances behind a load balancer. Each instance opens up to
 `2 × (DB_POOL_SIZE + DB_NESTED_POOL_SIZE)` database connections (30 with the defaults); keep the sum
 of all instances below PostgreSQL's `max_connections` (100 by default).
 
+## Monitoring with OpenTelemetry
+
+(0.72; APEX: Workspace Utilities → OpenTelemetry) pgkiln sends **traces, logs and metrics** to your own monitoring
+over OTLP, the OpenTelemetry protocol: Grafana (Tempo, Loki, Mimir), Jaeger, Datadog, Honeycomb, New Relic, Elastic,
+OCI APM and the rest, normally through an [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/). It sends
+OTLP over HTTP as JSON (`http/json`), which every Collector accepts on port 4318. No agent and no extra package are
+needed, and nothing runs while no collector is set.
+
+Administrators set it up in Workspace utilities → **OpenTelemetry** (`/builder/telemetry`), or with the standard
+`OTEL_*` variables of the [configuration reference](#configuration-reference) (a value set in the builder wins):
+
+- **Collector URL**: the OTLP/HTTP base address; `/v1/traces`, `/v1/logs` and `/v1/metrics` are added.
+  **Headers** (an API key) are stored encrypted with `PGKILN_SECRET_KEY` (rotated by `pgkiln admin rotate-key`)
+  and never shown again, only their names. **Send a test span** sends one span and shows what the collector answered;
+  *Status* shows what each server process sent, dropped, and the last problem.
+- **What is traced.** Every application request is a trace: a server span `GET /a/:alias/:page` with the application,
+  page, HTTP status, route and the user (see below), and a child span for **each SQL statement** (`db.query.text`,
+  rows, SQLSTATE on errors; transaction control left out), **each outgoing web call** (REST data sources, `invoke_api`,
+  `meta.web_request`, web credential token requests: `HTTP GET` with the host, path and status; a 4xx/5xx answer is
+  an error span), **each AI request** (`gen_ai.*`: provider, model, input and output tokens) and **each AI tool call**
+  of an assistant or agent. A W3C `traceparent` header goes along with every outgoing web call, so the called service
+  joins the same trace, and an incoming `traceparent` (from a proxy or another service) is continued. Background work
+  has traces of its own: each **mail** sending attempt and each **workflow step** (with the `invoke_api` call inside it).
+  *Trace builder requests too* adds `/builder` requests (to the instance's collector only).
+- **Sample ratio**: the share of traces sent (0 to 1). pgkiln keeps a trace's spans until it ends, so a trace with an
+  error (an HTTP 5xx, a failing statement, a REST call answered with 500) is **always** sent.
+- **Values stay out** unless *Include values* is on: SQL statements are sent with their literals replaced by `?`
+  (`where ename = ?`), bind values, the query string, the full URL of outgoing calls and AI tool arguments are left
+  out, and quoted values, `Key (col)=(value)`, e-mail addresses and long numbers are removed from error messages.
+  With *Include values*, statements go as run, with `db.query.parameter.N`.
+- **User**: *Hashed* (the default) sends `enduser.id` as a keyed hash (the same user always gives the same value,
+  the name is not sent), *User name* sends the name, *Not sent* leaves it out.
+- **Logs**: errors (also unhandled server errors), refused access and failed or locked sign-ins, as log records tied
+  to the request's trace, with the application and page.
+- **Metrics**, every 60 seconds (`OTEL_METRIC_EXPORT_INTERVAL`): `pgkiln.http.server.duration` (a histogram in
+  milliseconds per application, method and status class), `pgkiln.errors`, `pgkiln.access_denied`,
+  `pgkiln.signin_failures`, `pgkiln.ai.requests`, `pgkiln.ai.tokens` (per provider, model, input/output),
+  `pgkiln.mail.sent`, and the levels `pgkiln.mail.queue`, `pgkiln.jobs.queue`, `pgkiln.workflows`,
+  `pgkiln.ai.requests.pending`, `pgkiln.embeddings.queue`, `pgkiln.db.pool.connections`, `process.memory.rss`,
+  `process.uptime`; `pgkiln.telemetry.dropped` counts what could not be sent.
+- **Browser spans** (optional): a page whose trace is sent carries a signed `<meta name="pgkiln-trace">`, and `app.js`
+  reports the page's load timings and its Ajax calls to `POST /a/<alias>/telemetry`; pgkiln adds them to the page's
+  trace (*browser page load*, *browser POST*). The browser never sees the collector or its headers, and the token is
+  bound to the trace, the application and 30 minutes.
+- **Prometheus**: *Prometheus endpoint /metrics* (or `PGKILN_METRICS=on`) answers `GET /metrics` in the Prometheus
+  text format, for administrators signed in to the builder and for scrapers with the metrics token (*Make a token*,
+  shown once; or `PGKILN_METRICS_TOKEN`). It works without a collector.
+
+Each **workspace** chooses on its page → *Mail and network access* → **OpenTelemetry**
+(`/builder/workspaces/<id>/telemetry`, its administrators): *As the instance does* (the default), *Nothing* (its
+applications send nothing, not even to the instance's collector), or *Also to our own collector*: an OTLP/HTTP URL
+with its own headers and its own *Include values*, reached like any outgoing call of the workspace, through the
+installation's allow-list (`PGKILN_REST_ALLOWED_HOSTS`) and the workspace's [network access](03-builder.md#network-access)
+list. Its resource carries `pgkiln.workspace.id`. Metrics go to the instance's collector only.
+
+A Collector that sends traces to Jaeger, for a first look (`docker compose`):
+
+```yaml
+services:
+  jaeger:
+    image: jaegertracing/all-in-one:1.62.0     # the UI on http://localhost:16686
+    environment: { COLLECTOR_OTLP_ENABLED: "true" }
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.114.0
+    command: ["--config=/etc/otel.yaml"]
+    volumes: ["./otel.yaml:/etc/otel.yaml:ro"]
+    ports: ["4318:4318"]
+```
+
+```yaml
+# otel.yaml
+receivers:
+  otlp: { protocols: { http: { endpoint: 0.0.0.0:4318 } } }
+exporters:
+  otlp/jaeger: { endpoint: jaeger:4317, tls: { insecure: true } }
+  debug: {}
+service:
+  pipelines:
+    traces: { receivers: [otlp], exporters: [otlp/jaeger] }
+    logs: { receivers: [otlp], exporters: [debug] }
+    metrics: { receivers: [otlp], exporters: [debug] }
+```
+
+Set `OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318` (or the URL in the builder), open a page, and its trace shows
+in Jaeger under the service `pgkiln`. A Prometheus scrape job for `/metrics`:
+
+```yaml
+scrape_configs:
+  - job_name: pgkiln
+    metrics_path: /metrics
+    authorization: { credentials: "<the metrics token>" }
+    static_configs: [{ targets: ["apps.example.com:443"] }]
+    scheme: https
+```
+
+Every server process sends what it saw: with several instances, each is a `service.instance.id` (host and process).
+A collector that is down costs a failed request per batch every few seconds, never a user's request; what doesn't fit
+the queue (8,192 spans per collector) is dropped and counted.
+
 ## Backups and moving applications
 
 - **Everything** is in PostgreSQL: application definitions (`meta.*`), sessions and your data.

package/docs/guide/03-builder.md

@@ -1724,6 +1724,15 @@ doesn't allow:
   `PGKILN_REST_PRIVATE_HOSTS` lists it). The server resolves the host once and connects to the address it checked,
   with the host name kept for TLS, on every send and test.
 
+### OpenTelemetry
+
+Workspace utilities → **OpenTelemetry** (`/builder/telemetry`, administrators; APEX 24.2: Workspace Utilities →
+OpenTelemetry): traces, logs and metrics of the applications to an OpenTelemetry Collector, and `GET /metrics` for
+Prometheus. A workspace's page → *Mail and network access* → **OpenTelemetry** lets its administrators switch its
+applications off or send them to a collector of their own as well (through the allow-list and the workspace's
+[network access](#network-access) list). What is sent, the values left out and the settings are in
+[chapter 1](01-installation.md#monitoring-with-opentelemetry).
+
 ### Network access
 
 Administration menu → **Network access** (`/builder/network`; APEX 26.2: workspace network ACLs), for administrators

package/docs/guide/09-reference.md

@@ -649,6 +649,9 @@ pgkiln's icons; its filter box also searches the Lucide icons by name and search
 | `GET/POST /a/:alias/account`, `POST /a/:alias/account/password`, `POST /a/:alias/account/theme`, `POST /a/:alias/account/style` | My account, own password, the light/dark switch, the [style variant](14-globalization.md#style-variants-theme-roller) switch |
 | `POST /a/:alias/tz` | The browser's time zone for the session (automatic time zone; sent by `app.js`, CSRF token required) |
 | any page `?lang=xx` | Switch the language for the session |
+| `POST /a/:alias/telemetry` | (0.72) Browser spans of a page whose trace goes to an OpenTelemetry collector (sent by `app.js`; a signed token of the page, 204) |
+| `GET /metrics` | (0.72) Metrics in the Prometheus text format, when switched on: an administrator's builder session or `Authorization: Bearer <metrics token>` ([chapter 1](01-installation.md#monitoring-with-opentelemetry)) |
+| `GET /healthz` | `ok` when the database is reachable (503 otherwise) |
 | `/builder/...` | Builder |
 | PostgREST (separate service, `API_URL`) | REST API of each app's `api` schema, see [chapter 13](13-rest-api.md) |
 | `/static/...` | CSS, JavaScript, icons |

package/docs/guide/12-development.md

@@ -190,6 +190,16 @@ src/
   debug.ts                 debug messages: DebugLog (levels, timed steps, NOTICEs of meta.debug from appTx), started in
                            loadContext, stored after the response (onResponse hook → meta.debug_save), hourly purge
   icons.ts                 icon helper (sprite in public/icons.svg)
+  telemetry/               (sprint 139, AP-6) OpenTelemetry without the SDK: trace.ts (the current trace in AsyncLocalStorage,
+                           startTrace/endTrace, withSpan/rootSpan/recordSpan, traceApp, traceparent, a span per statement by
+                           patching pg.Client#query, sanitizeSql/redactText, counters and histograms), config.ts (the instance's
+                           settings: meta.setting otel_* over OTEL_* variables; workspaces' telemetry columns; app → workspace,
+                           cached 30 s), export.ts (a queue per collector, OTLP/HTTP JSON for traces, logs and metrics, values
+                           only where included, gauges from the queues, prometheusText(), sendTest()), http.ts (a trace per
+                           request, request metrics, GET /metrics, POST /a/:alias/telemetry), browser.ts (the page's signed
+                           <meta name="pgkiln-trace"> in layout.ts and relay() of what app.js reports). Spans come from
+                           webclient.ts (outgoing calls, traceparent), ai/service.ts logUsage, runtime/ai-agents.ts logToolCall,
+                           mail/mail.ts sendOne, workflow.ts runWorkflow; logs from session.ts logActivity
   history.ts               (sprint 138, AP-5) change history and checksums: meta.app_snapshot (the export after the last recorded
                            change), recordHistory() (export again, workingcopy.ts components() of both, each added/changed/deleted
                            component with its files before and after into meta.component_change; called by app-updates.ts onSend,
@@ -392,6 +402,9 @@ src/
     vector-providers.ts    (sprint 136, AP-1) Shared Components → Vector providers (/apps/:id/vector-providers: list, provider form,
                            test, delete) and Workspace utilities → Vector providers (/vector-providers: pgvector status, Create
                            extension for administrators, the workspace's providers and vector columns); pgvectorNote()
+    telemetry.ts           (sprint 139, AP-6) Workspace utilities → OpenTelemetry (/telemetry: the collector, signals, values, user,
+                           browser spans, /metrics and its token, status per collector, Send a test span; administrators) and a
+                           workspace's own (/workspaces/:id/telemetry: as the instance / off / own collector; its administrators)
     data-security.ts       (sprint 119) a table's Security: GET/POST /sql/objects/security?o= (policies, roles × columns, masks), Preview / Apply
     object-actions.ts      (sprint 94) an object's ⋯ menu (objectMenu) in the worksheet and the Object Browser: Quick DDL (objectDdl), Drop with
                            dependents, Add index / trigger / policy as drafts; GET /sql/objects/action, /sql/objects/panel (Open in a panel)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pgkiln",
-  "version": "0.71.0",
+  "version": "0.72.0",
   "description": "Open source low-code, SQL-driven application builder for PostgreSQL, inspired by Oracle APEX",
   "keywords": [
     "postgresql",

package/public/app.js

@@ -4362,3 +4362,53 @@ document.addEventListener('change', (e) => {
     });
   }
 })();
+
+// (sprint 139, AP-6) Browser spans: a page whose trace goes to an OpenTelemetry collector carries a
+// signed <meta name="pgkiln-trace">. Its load timings and its Ajax calls to the same application are
+// reported to pgkiln (POST /a/<alias>/telemetry), which adds them to the page's trace; the browser
+// never sees the collector. Nothing happens on pages without the meta.
+(() => {
+  const meta = document.querySelector('meta[name="pgkiln-trace"]');
+  const m = /^\/a\/[^/]+\//.exec(location.pathname);
+  if (!meta || !m || !window.fetch || !window.performance) return;
+  const token = meta.content;
+  const url = `${m[0]}telemetry`;
+  const origin = performance.timeOrigin || Date.now() - performance.now();
+  let queue = [];
+  const send = () => {
+    if (!queue.length) return;
+    const body = JSON.stringify({ t: token, spans: queue.splice(0, 50) });
+    const blob = new Blob([body], { type: 'application/json' });
+    if (!(navigator.sendBeacon && navigator.sendBeacon(url, blob)))
+      realFetch(url, { method: 'POST', body, headers: { 'content-type': 'application/json' }, keepalive: true, credentials: 'same-origin' }).catch(() => {});
+  };
+  const realFetch = window.fetch.bind(window);
+  window.fetch = function (input, init) {
+    const t0 = performance.now();
+    let path = '';
+    try {
+      const u = new URL(typeof input === 'string' ? input : input instanceof Request ? input.url : String(input), location.href);
+      if (u.origin === location.origin && u.pathname.startsWith(m[0]) && u.pathname !== url) path = u.pathname + u.search;
+    } catch {
+      // not a URL we report
+    }
+    const p = realFetch(input, init);
+    if (path) {
+      const method = String((init && init.method) || (input instanceof Request ? input.method : 'GET')).toUpperCase();
+      const done = (status) => {
+        queue.push({ k: 'ajax', path, method, status, start: origin + t0, dur: performance.now() - t0 });
+        if (queue.length >= 20) send();
+      };
+      p.then((r) => done(r.status), () => done(0));
+    }
+    return p;
+  };
+  window.addEventListener('load', () => setTimeout(() => {
+    const n = performance.getEntriesByType('navigation')[0];
+    if (n) queue.push({ k: 'load', start: origin + n.startTime, dur: (n.loadEventEnd || performance.now()) - n.startTime,
+      t: { ttfb: n.responseStart - n.startTime, dom_interactive: n.domInteractive - n.startTime, dom_complete: n.domComplete - n.startTime, load: n.loadEventEnd - n.startTime } });
+    send();
+  }, 0));
+  document.addEventListener('visibilitychange', () => { if (document.visibilityState === 'hidden') send(); });
+  window.addEventListener('pagehide', send);
+})();

package/public/builder.css

@@ -1645,3 +1645,6 @@ a.cp-type:hover, a.cp-type:focus-visible { border-color: var(--accent); }
 .util-icons code { font-size: 0.75rem; overflow-wrap: anywhere; }
 .ui-defaults :is(input, select, textarea) { width: 100%; min-width: 7rem; }
 .util-upgrade + .util-upgrade { margin-top: 1rem; }
+
+/* (sprint 139, AP-6) OpenTelemetry */
+.otel-problem { color: var(--danger); overflow-wrap: anywhere; }

package/src/ai/service.ts

@@ -3,6 +3,7 @@ import { decryptSecret } from '../secrets.ts';
 import { ClaudeProvider } from './anthropic.ts';
 import { OpenAiProvider } from './openai.ts';
 import { AiError, type AiProvider, type GenerateRequest, type GenerateResult, type ProviderName } from './types.ts';
+import { count, recordSpan } from '../telemetry/trace.ts';
 
 // AI services (migration 060; Gemini and OpenAI-compatible endpoints: 121): an administrator configures them in the
 // builder (Workspace utilities → AI services) and allows each application
@@ -115,6 +116,19 @@ export interface Usage {
 }
 
 export async function logUsage(info: Pick<CallInfo, 'appId' | 'pageNo' | 'user' | 'source'>, u: Usage) {
+  // (sprint 139, AP-6) the call as a span of the current trace (timed by the caller), tokens as metrics
+  if (u.status !== 'limited')
+    recordSpan(`AI ${u.service.provider} ${u.model}`.slice(0, 120), {
+      ms: u.ms,
+      attrs: {
+        'gen_ai.system': u.service.provider, 'gen_ai.request.model': u.model, 'gen_ai.operation.name': info.source === 'embedding' ? 'embeddings' : 'chat',
+        'gen_ai.usage.input_tokens': u.inputTokens, 'gen_ai.usage.output_tokens': u.outputTokens, 'pgkiln.ai.service': u.service.name, 'pgkiln.ai.source': info.source,
+      },
+      error: u.status === 'ok' ? null : `${u.status}${u.message ? `: ${u.message}` : ''}`,
+    });
+  count('pgkiln.ai.requests', { provider: u.service.provider, status: u.status });
+  count('pgkiln.ai.tokens', { provider: u.service.provider, model: u.model.slice(0, 100), type: 'input' }, u.inputTokens);
+  count('pgkiln.ai.tokens', { provider: u.service.provider, model: u.model.slice(0, 100), type: 'output' }, u.outputTokens);
   try {
     await owner.query(
       `insert into meta.ai_usage (app_id, page_no, username, service_id, service, provider, model, source, input_tokens, output_tokens, duration_ms, status, message)

package/src/app.ts

@@ -97,6 +97,10 @@ import { mailQueueRoutes } from './builder/mail-queue.ts';
 import { networkAccessRoutes } from './builder/network-access.ts';
 import { workspaceMailRoutes } from './builder/workspace-mail.ts';
 import { runtimeScope } from './db.ts';
+// (sprint 139, AP-6) OpenTelemetry: a trace per request, /metrics, the browser relay
+import { startTelemetry } from './telemetry/export.ts';
+import { telemetryHooks, telemetryRoutes } from './telemetry/http.ts';
+import { telemetryBuilderRoutes } from './builder/telemetry.ts';
 
 export async function buildApp(opts: { logger?: boolean } = {}) {
   // a database older than the code: apply the migrations (MIGRATE_ON_START=true)
@@ -117,6 +121,9 @@ export async function buildApp(opts: { logger?: boolean } = {}) {
     trustProxy: trustProxySetting(process.env.TRUST_PROXY),
   });
   securityHeaders(app);
+  // (sprint 139) before the workspace scope below, so the request's trace encloses everything
+  if (!pending.length) await startTelemetry();
+  telemetryHooks(app);
   if (pending.length) {
     // checked again (at most every 5 s) until someone migrates; then requests go through
     let checkedAt = Date.now();
@@ -279,6 +286,9 @@ export async function buildApp(opts: { logger?: boolean } = {}) {
   await app.register(dependencyRoutes);
   await app.register(upgradeRoutes);
   await app.register(uiDefaultRoutes);
+  // (sprint 139, AP-6) OpenTelemetry: Workspace utilities → OpenTelemetry, a workspace's own, /metrics, the browser relay
+  await app.register(telemetryRoutes);
+  await app.register(telemetryBuilderRoutes);
   app.get('/', async (_req, reply) => reply.redirect('/builder/home'));
   return app;
 }

package/src/builder/home.ts

@@ -300,6 +300,7 @@ export async function homeRoutes(app: FastifyInstance) {
         ${card(`${BASE}/vector-providers`, icon('search'), 'Vector providers', 'Embeddings for search by meaning: the applications\' vector providers, their vector columns, and whether pgvector is installed.')}
         ${(await isAdmin(s.username)) ? card(`${BASE}/ai`, icon('bolt'), 'AI services', 'Claude and OpenAI for the applications: models, keys, limits and usage (administrators).') : ''}
         ${(await isAdmin(s.username)) ? card(`${BASE}/mail`, icon('send'), 'Mail services', 'SMTP, Microsoft 365, Amazon SES, Resend, Postmark, SendGrid or Mailgun for the applications\' e-mail (administrators).') : ''}
+        ${(await isAdmin(s.username)) ? card(`${BASE}/telemetry`, icon('activity'), 'OpenTelemetry', 'Traces, logs and metrics to your own monitoring through an OpenTelemetry Collector; /metrics for Prometheus (administrators).') : ''}
         ${(await isAdmin(s.username)) ? card(`${BASE}/installation`, icon('history'), 'Installation', 'Version, install and upgrade runs, applied migrations (administrators).') : ''}
       </div></div>
       <aside class="ab-side" aria-label="About and reports">

package/src/builder/i18n/de.ts

@@ -1686,6 +1686,8 @@ export const de: BuilderMessages = {
   'wsp.mail_service': 'Mail-Dienst',
   'wsp.network': 'Netzwerkzugriff',
   'wsp.queue': 'Mail-Warteschlange',
+  'wsp.telemetry': 'OpenTelemetry',
+  'wsp.telemetry_text': 'Traces und Logs der Anwendungen dieses Workspace: {link} (wie die Instanz, aus, oder zusätzlich an einen eigenen Collector des Workspace).',
   'wsp.iso_short_on': 'Isoliert: Seine Anwendungen und sein SQL laufen auf eigenen Datenbank-Logins.',
   'wsp.iso_short_off': 'Gemeinsam.',
   'wsp.admins_change': 'Administratoren ändern das.',

package/src/builder/i18n/en.ts

@@ -1690,6 +1690,8 @@ export const en = {
   'wsp.mail_service': 'mail service',
   'wsp.network': 'network access',
   'wsp.queue': 'mail queue',
+  'wsp.telemetry': 'OpenTelemetry',
+  'wsp.telemetry_text': 'Traces and logs of this workspace\'s applications: {link} (as the instance, off, or also to a collector of the workspace\'s own).',
   'wsp.iso_short_on': 'Isolated: its applications and SQL run on database logins of its own.',
   'wsp.iso_short_off': 'Shared.',
   'wsp.admins_change': 'Administrators change it.',

package/src/builder/i18n/es.ts

@@ -1686,6 +1686,8 @@ export const es: BuilderMessages = {
   'wsp.mail_service': 'servicio de correo',
   'wsp.network': 'acceso a la red',
   'wsp.queue': 'cola de correo',
+  'wsp.telemetry': 'OpenTelemetry',
+  'wsp.telemetry_text': 'Trazas y registros de las aplicaciones de este espacio de trabajo: {link} (como la instancia, desactivado, o también a un recolector propio del espacio de trabajo).',
   'wsp.iso_short_on': 'Aislado: sus aplicaciones y su SQL se ejecutan con inicios de sesión de base de datos propios.',
   'wsp.iso_short_off': 'Compartido.',
   'wsp.admins_change': 'Los administradores lo cambian.',

package/src/builder/i18n/fr.ts

@@ -1686,6 +1686,8 @@ export const fr: BuilderMessages = {
   'wsp.mail_service': 'service de messagerie',
   'wsp.network': 'accès réseau',
   'wsp.queue': 'file des e-mails',
+  'wsp.telemetry': 'OpenTelemetry',
+  'wsp.telemetry_text': 'Traces et journaux des applications de cet espace de travail : {link} (comme l\'instance, désactivé, ou aussi vers un collecteur propre à l\'espace de travail).',
   'wsp.iso_short_on': 'Isolé : ses applications et son SQL s’exécutent sur des connexions de base de données propres.',
   'wsp.iso_short_off': 'Partagé.',
   'wsp.admins_change': 'Les administrateurs le modifient.',

package/src/builder/i18n/it.ts

@@ -1686,6 +1686,8 @@ export const it: BuilderMessages = {
   'wsp.mail_service': 'servizio di posta',
   'wsp.network': 'accesso di rete',
   'wsp.queue': 'coda di posta',
+  'wsp.telemetry': 'OpenTelemetry',
+  'wsp.telemetry_text': 'Tracce e log delle applicazioni di questo workspace: {link} (come l\'istanza, disattivato, o anche verso un collector proprio del workspace).',
   'wsp.iso_short_on': 'Isolato: le sue applicazioni e il suo SQL vengono eseguiti su login del database propri.',
   'wsp.iso_short_off': 'Condiviso.',
   'wsp.admins_change': 'Lo modificano gli amministratori.',

package/src/builder/i18n/ja.ts

@@ -1686,6 +1686,8 @@ export const ja: BuilderMessages = {
   'wsp.mail_service': 'メール・サービス',
   'wsp.network': 'ネットワーク・アクセス',
   'wsp.queue': 'メール・キュー',
+  'wsp.telemetry': 'OpenTelemetry',
+  'wsp.telemetry_text': 'このワークスペースのアプリケーションのトレースとログ：{link}（インスタンスと同じ、オフ、またはワークスペース独自のコレクターにも送信）。',
   'wsp.iso_short_on': '分離: アプリケーションと SQL は独自のデータベース・ログインで実行されます。',
   'wsp.iso_short_off': '共有。',
   'wsp.admins_change': '管理者が変更します。',

package/src/builder/i18n/ko.ts

@@ -1686,6 +1686,8 @@ export const ko: BuilderMessages = {
   'wsp.mail_service': '메일 서비스',
   'wsp.network': '네트워크 액세스',
   'wsp.queue': '메일 대기열',
+  'wsp.telemetry': 'OpenTelemetry',
+  'wsp.telemetry_text': '이 작업 영역 애플리케이션의 트레이스와 로그: {link} (인스턴스와 동일, 끔, 또는 작업 영역 자체 수집기로도 전송).',
   'wsp.iso_short_on': '격리됨: 애플리케이션과 SQL이 자체 데이터베이스 로그인으로 실행됩니다.',
   'wsp.iso_short_off': '공유.',
   'wsp.admins_change': '관리자가 변경합니다.',

package/src/builder/i18n/nl.ts

@@ -1686,6 +1686,8 @@ export const nl: BuilderMessages = {
   'wsp.mail_service': 'maildienst',
   'wsp.network': 'netwerktoegang',
   'wsp.queue': 'mailwachtrij',
+  'wsp.telemetry': 'OpenTelemetry',
+  'wsp.telemetry_text': 'Traces en logs van de applicaties van deze werkruimte: {link} (zoals de instantie, uit, of ook naar een eigen collector van de werkruimte).',
   'wsp.iso_short_on': 'Geïsoleerd: de applicaties en SQL draaien op eigen databaselogins.',
   'wsp.iso_short_off': 'Gedeeld.',
   'wsp.admins_change': 'Beheerders wijzigen dit.',

package/src/builder/i18n/pt.ts

@@ -1686,6 +1686,8 @@ export const pt: BuilderMessages = {
   'wsp.mail_service': 'serviço de e-mail',
   'wsp.network': 'acesso à rede',
   'wsp.queue': 'fila de e-mails',
+  'wsp.telemetry': 'OpenTelemetry',
+  'wsp.telemetry_text': 'Traces e logs das aplicações deste espaço de trabalho: {link} (como a instância, desligado, ou também para um coletor próprio do espaço de trabalho).',
   'wsp.iso_short_on': 'Isolado: suas aplicações e seu SQL são executados em logins de banco de dados próprios.',
   'wsp.iso_short_off': 'Compartilhado.',
   'wsp.admins_change': 'Os administradores o alteram.',

package/src/builder/i18n/zh-cn.ts

@@ -1686,6 +1686,8 @@ export const zhCN: BuilderMessages = {
   'wsp.mail_service': '邮件服务',
   'wsp.network': '网络访问',
   'wsp.queue': '邮件队列',
+  'wsp.telemetry': 'OpenTelemetry',
+  'wsp.telemetry_text': '此工作区应用程序的跟踪和日志：{link}（与实例相同、关闭，或同时发送到工作区自己的收集器）。',
   'wsp.iso_short_on': '已隔离：其应用程序和 SQL 在专属的数据库登录上运行。',
   'wsp.iso_short_off': '共享。',
   'wsp.admins_change': '由管理员更改。',

package/src/builder/i18n/zh-tw.ts

@@ -1686,6 +1686,8 @@ export const zhTW: BuilderMessages = {
   'wsp.mail_service': '郵件服務',
   'wsp.network': '網路存取',
   'wsp.queue': '郵件佇列',
+  'wsp.telemetry': 'OpenTelemetry',
+  'wsp.telemetry_text': '此工作區應用程式的追蹤與日誌：{link}（與執行個體相同、關閉，或同時傳送到工作區自己的收集器）。',
   'wsp.iso_short_on': '已隔離：其應用程式與 SQL 以專屬的資料庫登入執行。',
   'wsp.iso_short_off': '共用。',
   'wsp.admins_change': '由管理員變更。',