pgkiln 0.31.0 → 0.35.0

package/db/migrations/082_app_backups.sql

@@ -0,0 +1,73 @@
+-- =====================================================================
+-- 082: automatic application backups (APEX: Application Backups)
+--
+-- meta.app_backup keeps exports of an application: one before every
+-- import that replaces it (pgkiln import --replace, the MCP import, a
+-- working copy's merge, a restore), one a day when it changed since the
+-- last backup (the scheduler), and the ones a developer makes. The newest
+-- meta.app.backups_keep (default 25, 0 = none) are kept per application.
+-- The builder lists, downloads, restores and deletes them (Application →
+-- Backups); a restore is an import --replace of the backup, so the version
+-- it replaces is backed up too. Installation data: not exported, kept by
+-- an import over the application.
+-- =====================================================================
+
+create table meta.app_backup (
+  id         bigint generated always as identity primary key,
+  app_id     int not null references meta.app on delete cascade,
+  created_at timestamptz not null default now(),
+  created_by text,
+  reason     text not null check (reason in ('import', 'merge', 'restore', 'daily', 'manual')),
+  note       text check (length(note) <= 500),
+  doc        jsonb not null,
+  -- md5 of the export, so a daily backup of an unchanged application is skipped
+  doc_hash   text not null,
+  size       int not null
+);
+create index app_backup_app on meta.app_backup (app_id, created_at desc);
+revoke all on meta.app_backup from public;
+comment on table meta.app_backup is 'Application backups (082): exports kept by this installation, never exported';
+
+alter table meta.app add column backups_keep int not null default 25 check (backups_keep between 0 and 500);
+
+-- an export made before 082 has no backups_keep: NULL becomes 25
+create function meta.app_backups_keep_default() returns trigger
+language plpgsql set search_path = pg_catalog as $$
+begin
+  new.backups_keep := coalesce(new.backups_keep, 25);
+  return new;
+end $$;
+create trigger app_backups_keep before insert or update on meta.app
+  for each row execute function meta.app_backups_keep_default();
+
+-- Back an application up (the server and the command line call it as the owner). A daily
+-- backup of an application that didn't change since its newest backup is skipped (NULL).
+create function meta.backup_app(p_app_id int, p_reason text, p_user text default null, p_note text default null)
+returns bigint
+language plpgsql volatile set search_path = meta, pg_catalog as $$
+declare
+  v_alias text;
+  v_keep  int;
+  v_doc   jsonb;
+  v_hash  text;
+  v_id    bigint;
+begin
+  select alias, backups_keep into v_alias, v_keep from meta.app where id = p_app_id;
+  if v_alias is null or v_keep = 0 then
+    return null;
+  end if;
+  v_doc := meta.export_app(v_alias);
+  v_hash := md5(v_doc::text);
+  if p_reason = 'daily' and v_hash = (select doc_hash from meta.app_backup where app_id = p_app_id order by created_at desc, id desc limit 1) then
+    return null;
+  end if;
+  insert into meta.app_backup (app_id, created_by, reason, note, doc, doc_hash, size)
+  values (p_app_id, p_user, p_reason, left(p_note, 500), v_doc, v_hash, octet_length(v_doc::text))
+  returning id into v_id;
+  delete from meta.app_backup
+   where app_id = p_app_id
+     and id not in (select id from meta.app_backup where app_id = p_app_id order by created_at desc, id desc limit v_keep);
+  return v_id;
+end
+$$;
+revoke all on function meta.backup_app(int, text, text, text) from public;

package/db/migrations/083_availability.sql

@@ -0,0 +1,156 @@
+-- =====================================================================
+-- 083: application availability and build status (APEX: Application
+-- Availability and Build Status)
+--
+-- meta.app.availability:
+--   available   everyone who may sign in (the default)
+--   developers  only developers with a builder session in the same browser
+--               (the developer toolbar's token, migration 079)
+--   restricted  only the users in availability_users (and developers)
+--   unavailable nobody: every page shows availability_message (or a default)
+-- Pages, file downloads, REST modules and PostgREST (meta.api_check) follow
+-- it; automations, workflows, background jobs and queued mail keep running.
+--
+-- meta.app.build_status 'run_only': the builder refuses changes to the
+-- application (as if it were locked), except to its availability and build
+-- status. The command line's import --replace (deployment) still works.
+--
+-- Both belong to the installation: export_app leaves them out (redefined from
+-- 081), an import of a new application gets the defaults, and an import over
+-- an application keeps its values (src/cli/replace.ts).
+-- =====================================================================
+
+alter table meta.app
+  add column availability text not null default 'available' check (availability in ('available', 'developers', 'restricted', 'unavailable')),
+  add column availability_message text check (length(availability_message) <= 2000),
+  add column availability_users text[] not null default '{}',
+  add column build_status text not null default 'run_and_build' check (build_status in ('run_and_build', 'run_only'));
+
+-- an export has none of these: NULL becomes the default
+create function meta.app_availability_default() returns trigger
+language plpgsql set search_path = pg_catalog as $$
+begin
+  new.availability := coalesce(new.availability, 'available');
+  new.availability_users := coalesce(new.availability_users, '{}');
+  new.build_status := coalesce(new.build_status, 'run_and_build');
+  return new;
+end $$;
+create trigger app_availability before insert or update on meta.app
+  for each row execute function meta.app_availability_default();
+
+comment on column meta.app.availability is 'available, developers, restricted or unavailable (083)';
+comment on column meta.app.build_status is 'run_and_build, or run_only: the builder refuses changes (083)';
+
+-- ---------------------------------------------------------------- export
+-- Same as 081, without the availability and build status.
+create or replace function meta.export_app(p_alias text) returns jsonb
+language sql stable set search_path = meta, pg_catalog as $$
+  select jsonb_build_object(
+    'format', 'pgkiln/2',
+    -- (083) availability and build status belong to the installation
+    'app', to_jsonb(a) - 'id' - 'created_at' - 'updated_at' - 'availability' - 'availability_message' - 'availability_users' - 'build_status',
+    'authz_schemes', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.authz_scheme x where x.app_id = a.id), '[]'),
+    'app_items', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.app_item x where x.app_id = a.id), '[]'),
+    'app_processes', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.seq, x.id) from meta.app_process x where x.app_id = a.id), '[]'),
+    'lovs', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.lov x where x.app_id = a.id), '[]'),
+    'group_roles', coalesce((select jsonb_agg(to_jsonb(x) - 'app_id' order by x.group_name, x.role) from meta.app_group_role x where x.app_id = a.id), '[]'),
+    'text_messages', coalesce((select jsonb_agg(to_jsonb(x) - 'app_id' order by x.name, x.language) from meta.text_message x where x.app_id = a.id), '[]'),
+    'translations', coalesce((select jsonb_agg(to_jsonb(x) - 'app_id' order by x.language, x.source) from meta.translation x where x.app_id = a.id), '[]'),
+    'report_layouts', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' - 'logo' || jsonb_build_object('logo', encode(x.logo, 'base64')) order by x.name)
+                                  from meta.report_layout x where x.app_id = a.id), '[]'),
+    'automations', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' - 'next_run_at' - 'last_run_at' - 'last_status' - 'code' order by x.name)
+                               from meta.automation x where x.app_id = a.id), '[]'),
+    -- (044) the actions of the automations, by automation name
+    'automation_actions', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.automation_name collate "C", x.seq, x.name collate "C")
+                                      from meta.automation_action x where x.app_id = a.id), '[]'),
+    'document_templates', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.document_template x where x.app_id = a.id), '[]'),
+    'task_definitions', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.task_definition x where x.app_id = a.id), '[]'),
+    'workflow_definitions', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.workflow_definition x where x.app_id = a.id), '[]'),
+    'rest_modules', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.rest_module x where x.app_id = a.id), '[]'),
+    'template_components', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.static_id) from meta.template_component x where x.app_id = a.id), '[]'),
+    'build_options', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.build_option x where x.app_id = a.id), '[]'),
+    -- (030) secrets never leave the installation: they are entered again after an import
+    -- (050) nor do the password and refresh token of the OAuth2 flows, or the token state
+    'web_credentials', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' - 'secret_enc' - 'password_enc' - 'refresh_token_enc' - 'token_refreshed_at' order by x.name)
+                                   from meta.web_credential x where x.app_id = a.id), '[]'),
+    -- (050) the synchronisation's state (next and last run) belongs to the installation
+    'rest_sources', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' - 'sync_next_at' - 'sync_last_at' - 'sync_last_status' order by x.name)
+                                from meta.rest_source x where x.app_id = a.id), '[]'),
+    -- (041) data load definitions
+    'data_load_definitions', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.data_load_def x where x.app_id = a.id), '[]'),
+    -- (042) lists; entries keep their ids so parents can be remapped on import
+    'lists', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.list x where x.app_id = a.id), '[]'),
+    'list_entries', coalesce((select jsonb_agg(to_jsonb(x) - 'app_id' order by x.list_name, x.parent_id nulls first, x.seq, x.id) from meta.list_entry x where x.app_id = a.id), '[]'),
+    -- (042) supporting objects: install, upgrade and deinstall scripts (never run on import)
+    'supporting_scripts', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.supporting_script x where x.app_id = a.id), '[]'),
+    -- (069) plug-ins (their files are static files, their templates template components)
+    'plugins', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.plugin x where x.app_id = a.id), '[]'),
+    -- (080) search configurations
+    'search_configs', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.search_config x where x.app_id = a.id), '[]'),
+    -- (081) e-mail templates
+    'email_templates', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' order by x.name) from meta.email_template x where x.app_id = a.id), '[]'),
+    -- (068) static application files (JavaScript, CSS, images, fonts) as base64
+    'static_files', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'app_id' - 'content' - 'updated_at' || jsonb_build_object('content', translate(encode(x.content, 'base64'), E'\n', '')) order by x.name)
+                                from meta.static_file x where x.app_id = a.id), '[]'),
+    -- nav entries and regions keep their ids, so parents and references can be remapped on import
+    'nav', coalesce((select jsonb_agg(to_jsonb(x) - 'app_id' order by x.parent_id nulls first, x.seq, x.id) from meta.nav_entry x where x.app_id = a.id), '[]'),
+    'pages', coalesce((
+      select jsonb_agg(to_jsonb(p) - 'id' - 'app_id' || jsonb_build_object(
+        'regions', coalesce((select jsonb_agg(to_jsonb(r) - 'page_id' order by r.seq, r.id) from meta.region r where r.page_id = p.id), '[]'),
+        'items', coalesce((select jsonb_agg(to_jsonb(i) - 'id' - 'page_id' order by i.seq, i.id) from meta.item i where i.page_id = p.id), '[]'),
+        'buttons', coalesce((select jsonb_agg(to_jsonb(b) - 'id' - 'page_id' order by b.seq, b.id) from meta.button b where b.page_id = p.id), '[]'),
+        'dynamic_actions', coalesce((select jsonb_agg(to_jsonb(d) - 'id' - 'page_id' order by d.seq, d.id) from meta.dynamic_action d where d.page_id = p.id), '[]'),
+        'validations', coalesce((select jsonb_agg(to_jsonb(v) - 'id' - 'page_id' order by v.seq, v.id) from meta.validation v where v.page_id = p.id), '[]'),
+        'processes', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'page_id' order by x.seq, x.id) from meta.process x where x.page_id = p.id), '[]'),
+        'computations', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'page_id' order by x.seq, x.id) from meta.computation x where x.page_id = p.id), '[]'),
+        'branches', coalesce((select jsonb_agg(to_jsonb(x) - 'id' - 'page_id' order by x.seq, x.id) from meta.branch x where x.page_id = p.id), '[]')
+      ) order by p.page_no)
+      from meta.page p where p.app_id = a.id), '[]'))
+  from meta.app a
+  where a.alias = p_alias
+$$;
+
+-- ---------------------------------------------------------------- PostgREST
+-- Same as 011, plus the application's availability.
+create or replace function meta.api_check() returns void
+language plpgsql stable security definer set search_path = meta, pg_catalog as $$
+declare
+  v_claims jsonb := meta.jwt_claims();
+  v_role   text := current_setting('role', true);   -- the SET ROLE PostgREST did
+  v_app    meta.app;
+  v_acc    meta.account;
+begin
+  if v_claims is null or v_role is null or v_role in ('none', 'pgkiln_anon') then
+    return;
+  end if;
+  select * into v_app from meta.app a where a.alias = v_claims->>'app';
+  if not found or v_app.api_role is distinct from v_role then
+    raise exception 'This token is not valid for an application with API role %.', v_role
+      using errcode = 'PT401';
+  end if;
+  -- (083) the application's availability: unavailable or for developers only answers 503;
+  -- restricted only for the users it names (never for an OAuth client)
+  if v_app.availability in ('unavailable', 'developers') then
+    raise exception '%', coalesce(nullif(v_app.availability_message, ''), 'The application is not available at the moment.') using errcode = 'PT503';
+  end if;
+  if v_app.availability = 'restricted'
+     and (v_claims ? 'client_id' or not lower(meta.app_user()) = any (select lower(u) from unnest(v_app.availability_users) u)) then
+    raise exception 'The application is not available to this account at the moment.' using errcode = 'PT403';
+  end if;
+  if v_claims ? 'client_id' then
+    if not exists (select 1 from meta.api_client c
+                    where c.client_id = v_claims->>'client_id' and c.app_id = v_app.id and c.active) then
+      raise exception 'The OAuth client of this token does not exist or was revoked.' using errcode = 'PT401';
+    end if;
+    return;
+  end if;
+  select * into v_acc from meta.account ac where lower(ac.username) = lower(meta.app_user());
+  if not found or not v_acc.active then
+    raise exception 'The account of this token does not exist or is inactive.' using errcode = 'PT403';
+  end if;
+  if v_app.access_control <> 'any_user'
+     and not exists (select 1 from meta.app_access aa where aa.app_id = v_app.id and aa.account_id = v_acc.id) then
+    raise exception 'The account of this token has no access to the application.' using errcode = 'PT403';
+  end if;
+end
+$$;

package/db/migrations/084_feedback.sql

@@ -0,0 +1,46 @@
+-- =====================================================================
+-- 084: feedback (APEX: Feedback, Team Development → Feedback)
+--
+-- With meta.app.feedback on, the application's navigation bar has a
+-- "Feedback" link: users send a comment, an idea or a problem (with an
+-- optional rating) about the page they came from, and see their earlier
+-- feedback with the developers' answers. Developers read, answer and
+-- close it in the builder (Application → Feedback). The setting travels
+-- with the export; the feedback itself is installation data.
+-- =====================================================================
+
+alter table meta.app add column feedback boolean not null default false;
+
+-- an export made before 084 has no feedback setting: NULL becomes false
+create function meta.app_feedback_default() returns trigger
+language plpgsql set search_path = pg_catalog as $$
+begin
+  new.feedback := coalesce(new.feedback, false);
+  return new;
+end $$;
+create trigger app_feedback before insert or update on meta.app
+  for each row execute function meta.app_feedback_default();
+
+create table meta.feedback (
+  id           bigint generated always as identity primary key,
+  app_id       int not null references meta.app on delete cascade,
+  page_no      int,
+  username     text not null,
+  session_id   uuid,
+  kind         text not null check (kind in ('comment', 'idea', 'problem')),
+  rating       smallint check (rating between 1 and 5),
+  body         text not null check (length(body) between 1 and 4000),
+  user_agent   text check (length(user_agent) <= 300),
+  status       text not null default 'open' check (status in ('open', 'in_progress', 'done', 'declined')),
+  response     text check (length(response) <= 4000),
+  responded_by text,
+  responded_at timestamptz,
+  created_at   timestamptz not null default now()
+);
+create index feedback_app on meta.feedback (app_id, created_at desc);
+revoke all on meta.feedback from public;
+-- the runtime records feedback and shows users their own (src/runtime/feedback.ts); developers answer
+grant select, insert on meta.feedback to pgkiln_runtime;
+
+comment on table meta.feedback is 'Feedback of application users and the developers'' answers (084); not exported';
+comment on column meta.app.feedback is 'Users can send feedback from the navigation bar (084)';

package/db/migrations/085_page_behaviour.sql

@@ -0,0 +1,65 @@
+-- =====================================================================
+-- 085: page behaviour (sprint 52)
+--
+-- Dynamic actions (APEX: events and actions that were missing):
+--   events  after_refresh (a region was refreshed or loaded; trigger: region
+--           ids or static ids), selection_change (rows selected in a report
+--           or grid; trigger: region ids or static ids), focus and blur (Get /
+--           Lose Focus of items), key_release (items), before_submit (the page
+--           is about to be submitted; a declined Confirm stops it), custom
+--           (trigger: event names, raised with pgkiln.trigger(name, value))
+--   actions confirm (a question; No stops the actions after it), close_dialog
+--           (close the modal page and refresh its parent), cancel_dialog,
+--           clear (empty the affected items), open_region / close_region,
+--           download (runs a download process of the page by name), print,
+--           current_position (latitude,longitude into the affected item),
+--           share (the device's share sheet, or the link copied)
+--   The actions of one event and trigger now run one after the other, in
+--   sequence order (APEX: the true actions of a dynamic action).
+--
+-- Ajax callbacks (APEX: On Demand processes, apex.server.process): page and
+-- application processes with point "ajax" run only when JavaScript calls
+-- them by name: POST /a/<alias>/<page>/ajax/<name> (src/runtime/ajax.ts).
+--
+-- meta.app.error_function (APEX: Error Handling Function): a function of the
+-- application, function(jsonb) returns jsonb, that may rewrite every error
+-- message shown to users (src/runtime/context.ts publicError).
+--
+-- meta.app.session_warning_seconds (APEX: Session timeout warning): seconds
+-- before the session ends that the page warns and offers to stay signed in
+-- (0: no warning).
+-- =====================================================================
+
+alter table meta.dynamic_action drop constraint dynamic_action_event_check;
+alter table meta.dynamic_action add constraint dynamic_action_event_check check (event in (
+  'change', 'click', 'load', 'dialog_closed', 'after_refresh', 'selection_change', 'focus', 'blur', 'key_release', 'before_submit', 'custom'));
+
+alter table meta.dynamic_action drop constraint dynamic_action_action_check;
+alter table meta.dynamic_action add constraint dynamic_action_action_check check (action in (
+  'show', 'hide', 'enable', 'disable', 'set_value', 'execute_sql', 'refresh_region', 'refresh_item', 'alert', 'submit',
+  'set_focus', 'add_class', 'remove_class', 'show_success', 'show_error', 'clear_errors', 'ai_generate', 'execute_javascript', 'plugin',
+  'push_subscribe', 'confirm', 'close_dialog', 'cancel_dialog', 'clear', 'open_region', 'close_region', 'download', 'print',
+  'current_position', 'share'));
+
+alter table meta.process drop constraint process_point_check;
+alter table meta.process add constraint process_point_check check (point in ('submit', 'load', 'ajax'));
+alter table meta.app_process drop constraint app_process_point_check;
+alter table meta.app_process add constraint app_process_point_check check (point in ('after_login', 'before_page', 'ajax'));
+
+alter table meta.app
+  -- schema.function: called as the application's role with a jsonb error, returns jsonb (or null)
+  add column error_function text check (error_function ~ '^[A-Za-z_][A-Za-z0-9_$]{0,62}\.[A-Za-z_][A-Za-z0-9_$]{0,62}$'),
+  add column session_warning_seconds int not null default 120 check (session_warning_seconds between 0 and 3600);
+
+-- an export made before 085 has no session warning: NULL becomes 120
+create function meta.app_session_warning_default() returns trigger
+language plpgsql set search_path = pg_catalog as $$
+begin
+  new.session_warning_seconds := coalesce(new.session_warning_seconds, 120);
+  return new;
+end $$;
+create trigger app_session_warning before insert or update on meta.app
+  for each row execute function meta.app_session_warning_default();
+
+comment on column meta.app.error_function is 'schema.function(jsonb) returns jsonb that rewrites error messages (085)';
+comment on column meta.app.session_warning_seconds is 'Warn this many seconds before the session ends; 0: no warning (085)';