underpost 3.2.80 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/.github/workflows/ghpkg.ci.yml +7 -1
  2. package/.github/workflows/pwa-microservices-template-page.cd.yml +1 -16
  3. package/.github/workflows/pwa-microservices-template-test.ci.yml +1 -1
  4. package/.github/workflows/release.cd.yml +1 -9
  5. package/CHANGELOG.md +291 -1
  6. package/CLI-HELP.md +174 -23
  7. package/README.md +5 -2
  8. package/bin/build.js +7 -5
  9. package/bin/deploy.js +19 -17
  10. package/deploy/lib/logging.sh +96 -0
  11. package/deploy/pwa-microservices-template/deploy.sh +72 -0
  12. package/deploy/release/deploy.sh +62 -0
  13. package/docker-compose.yml +1 -1
  14. package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +5 -1
  15. package/manifests/cronjobs/dd-cron/dd-cron-dns.yaml +1 -1
  16. package/manifests/cronjobs/dd-cron/dd-cron-vultr.yaml +52 -0
  17. package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
  18. package/manifests/deployment/playwright/deployment.yaml +1 -1
  19. package/manifests/mongodb/kustomization.yaml +4 -1
  20. package/manifests/mongodb/statefulset.yaml +4 -0
  21. package/manifests/mongodb/storage-class.yaml +9 -2
  22. package/package.json +19 -19
  23. package/scripts/audit-selinux.sh +64 -0
  24. package/scripts/coverall-test.sh +24 -0
  25. package/scripts/gpu-diag.sh +0 -0
  26. package/scripts/ip-info.sh +0 -0
  27. package/scripts/k3s-node-setup.sh +18 -15
  28. package/scripts/kubeadm-node-setup.sh +12 -23
  29. package/scripts/link-local-underpost-cli.sh +0 -0
  30. package/scripts/lxd-vm-setup.sh +0 -0
  31. package/scripts/maas-nat-firewalld.sh +0 -0
  32. package/scripts/nat-iptables.sh +12 -4
  33. package/scripts/rhel-grpc-setup.sh +0 -0
  34. package/scripts/rocky-kickstart.sh +25 -9
  35. package/scripts/test-monitor.sh +4 -3
  36. package/src/cli/baremetal.js +1 -2
  37. package/src/cli/cloud-init.js +1 -1
  38. package/src/cli/cluster.js +786 -96
  39. package/src/cli/db.js +11 -4
  40. package/src/cli/deploy.js +1698 -177
  41. package/src/cli/docker-compose.js +19 -178
  42. package/src/cli/env.js +1 -1
  43. package/src/cli/image.js +15 -7
  44. package/src/cli/index.js +245 -44
  45. package/src/cli/ipfs.js +82 -11
  46. package/src/cli/lxd.js +1 -1
  47. package/src/cli/monitor.js +2 -2
  48. package/src/cli/release.js +57 -22
  49. package/src/cli/repository.js +12 -10
  50. package/src/cli/run.js +2195 -427
  51. package/src/cli/secrets.js +969 -0
  52. package/src/cli/ssh.js +206 -105
  53. package/src/cli/system.js +26 -13
  54. package/src/cli/test.js +1 -1
  55. package/src/cli/vultr.js +583 -0
  56. package/src/cli/wireguard.js +2125 -0
  57. package/src/client-builder/client-build.js +102 -13
  58. package/src/client-builder/ssr.js +27 -73
  59. package/src/db/mongo/MongoBootstrap.js +295 -54
  60. package/src/db/mongo/MongooseDB.js +51 -32
  61. package/src/index.js +25 -1
  62. package/src/projects/underpost/catalog-underpost.js +4 -1
  63. package/src/server/backup.js +1 -1
  64. package/src/server/conf.js +1216 -168
  65. package/src/server/cri.js +70 -0
  66. package/src/server/cron.js +249 -51
  67. package/src/server/dns.js +100 -6
  68. package/src/server/environment.js +98 -0
  69. package/src/server/forward-proxy.js +549 -0
  70. package/src/server/middlewares.js +56 -1
  71. package/src/server/process.js +0 -1
  72. package/src/server/selinux.js +185 -0
  73. package/src/server/systemd.js +205 -0
  74. package/src/server/underpost-compression.js +186 -0
  75. package/src/server/underpost-gateway.js +1083 -0
  76. package/src/server/underpost-ingress.js +380 -0
  77. package/test/cluster-instances.test.js +435 -0
  78. package/test/deploy-node-placement.test.js +45 -0
  79. package/test/instance-traffic-plan.test.js +710 -0
  80. package/test/selinux.test.js +71 -0
  81. package/test/sops-secret-store.test.js +612 -0
  82. package/test/underpost-gateway.test.js +510 -0
  83. package/test/underpost-ingress.test.js +305 -0
  84. package/test/wireguard-edge.test.js +1177 -0
@@ -23,12 +23,81 @@ import { shellExec } from '../server/process.js';
23
23
  import { SitemapStream, streamToPromise } from 'sitemap';
24
24
  import { Readable } from 'stream';
25
25
  import { buildIcons } from './client-icons.js';
26
+ import { statusPageBuildSegment } from '../server/underpost-gateway.js';
26
27
  import Underpost from '../index.js';
27
28
  import { buildDocs } from './client-build-docs.js';
28
29
  import { ssrFactory } from './ssr.js';
29
30
 
30
31
  // Static Site Generation (SSG)
31
32
 
33
+ const STATUS_PAGE_VIEW_PATH = /^\/([1-5]\d{2})$/;
34
+
35
+ // Views a route intercepts before the workload sees the request. They are
36
+ // declared by the flag that already marks them as the app's default for that
37
+ // condition, so a new one becomes edge-served by adding its flag here rather
38
+ // than by naming its path in a second place.
39
+ const INTERCEPT_VIEW_FLAGS = ['maintenanceDefault', 'offlineDefault'];
40
+
41
+ /**
42
+ * Resolves the SSR views that render an HTTP status page into the static
43
+ * artifacts they build to. A view is a status page when its route path is a
44
+ * bare status code (`/404`, `/500`, `/503`), which the build writes to
45
+ * `<path>/index.html` inside the served bundle.
46
+ *
47
+ * Single source of truth for PWA status-page routing: this build writes the
48
+ * artifact, and `deploy --build-manifest` points its HTTPRoute rules at the
49
+ * same resolved URL — neither side hardcodes a status code.
50
+ * @function statusPageRoutesFactory
51
+ * @param {Array<object>} [views] - SSR view entries from `conf.ssr.json`.
52
+ * @param {string} [proxyPath] - The client's proxy sub-path (`/`, `/peer`, ...).
53
+ * @returns {Array<{status: string, routePath: string, indexUrl: string, title: string, client: string}>}
54
+ * One entry per status view, in declaration order.
55
+ * @memberof clientBuild
56
+ */
57
+ const statusPageRoutesFactory = ({ views = [], proxyPath = '/' } = {}) => {
58
+ const prefix = !proxyPath || proxyPath === '/' ? '' : proxyPath.replace(/\/$/, '');
59
+ return (Array.isArray(views) ? views : [])
60
+ .map((view) => ({ view, status: STATUS_PAGE_VIEW_PATH.exec(view?.path || '')?.[1] }))
61
+ .filter(({ status }) => status !== undefined)
62
+ .map(({ view, status }) => ({
63
+ status,
64
+ routePath: `${prefix}${view.path}`,
65
+ indexUrl: `${prefix}${view.path}/index.html`,
66
+ title: view.title,
67
+ client: view.client,
68
+ }));
69
+ };
70
+
71
+ /**
72
+ * Resolves the SSR views a gateway route intercepts and serves statically —
73
+ * the maintenance and offline documents. They carry no request-time logic, so
74
+ * the workload never needs to see them; the same build that writes the artifact
75
+ * hands `deploy --build-manifest` the URL its HTTPRoute rule targets.
76
+ * @function staticContextRoutesFactory
77
+ * @param {Array<object>} [views] - SSR view entries from `conf.ssr.json`.
78
+ * @param {string} [proxyPath] - The client's proxy sub-path (`/`, `/peer`, ...).
79
+ * @returns {Array<{context: string, routePath: string, indexUrl: string, title: string, client: string}>}
80
+ * One entry per intercepted view, in declaration order.
81
+ * @memberof clientBuild
82
+ */
83
+ const staticContextRoutesFactory = ({ views = [], proxyPath = '/' } = {}) => {
84
+ const prefix = !proxyPath || proxyPath === '/' ? '' : proxyPath.replace(/\/$/, '');
85
+ return (Array.isArray(views) ? views : [])
86
+ .filter(
87
+ (view) =>
88
+ view?.path &&
89
+ !STATUS_PAGE_VIEW_PATH.test(view.path) &&
90
+ INTERCEPT_VIEW_FLAGS.some((flag) => view[flag] === true),
91
+ )
92
+ .map((view) => ({
93
+ context: view.path.replace(/^\/+|\/+$/g, ''),
94
+ routePath: `${prefix}${view.path}`,
95
+ indexUrl: `${prefix}${view.path}/index.html`,
96
+ title: view.title,
97
+ client: view.client,
98
+ }));
99
+ };
100
+
32
101
  /**
33
102
  * Recursively copies files from source to destination, but only files that don't exist in destination.
34
103
  * @function copyNonExistingFiles
@@ -456,6 +525,7 @@ const defaultSitemapXsl = `<?xml version="1.0" encoding="UTF-8"?>
456
525
  * @param {string|number} options.split - Optional zip split size in MB.
457
526
  * @param {boolean} options.fullBuild - Whether to perform a full build.
458
527
  * @param {boolean} options.iconsBuild - Whether to build icons.
528
+ * @param {boolean} options.ssrOnly - Whether to rebuild only views defined in `conf.ssr.json`.
459
529
  * @returns {Promise<void>} - Promise that resolves when the build is complete.
460
530
  * @throws {Error} - If the build fails.
461
531
  * @memberof clientBuild
@@ -479,6 +549,7 @@ const buildClient = async (
479
549
  const packageData = JSON.parse(fs.readFileSync(`./package.json`, 'utf8'));
480
550
  const acmeChallengePath = `/.well-known/acme-challenge`;
481
551
  const publicPath = `./public`;
552
+ const ssrOnly = options.ssrOnly === true;
482
553
 
483
554
  /**
484
555
  * @async
@@ -622,11 +693,11 @@ const buildClient = async (
622
693
  ? `${directory}${acmeChallengePath}`
623
694
  : `${publicPath}/${host}${acmeChallengePath}`;
624
695
 
625
- if (!enableLiveRebuild) buildAcmeChallengePath(acmeChallengeFullPath);
696
+ if (!enableLiveRebuild && !ssrOnly) buildAcmeChallengePath(acmeChallengeFullPath);
626
697
 
627
698
  if (redirect || disabledRebuild) continue;
628
699
 
629
- if (fullBuildEnabled)
700
+ if (fullBuildEnabled && !ssrOnly)
630
701
  await fullBuild({
631
702
  path,
632
703
  logger,
@@ -641,7 +712,7 @@ const buildClient = async (
641
712
  publicCopyNonExistingFiles,
642
713
  });
643
714
 
644
- if (components)
715
+ if (!ssrOnly && components)
645
716
  for (const module of Object.keys(components)) {
646
717
  if (!fs.existsSync(`${rootClientPath}/components/${module}`))
647
718
  fs.mkdirSync(`${rootClientPath}/components/${module}`, { recursive: true });
@@ -664,7 +735,7 @@ const buildClient = async (
664
735
  }
665
736
  }
666
737
 
667
- if (services) {
738
+ if (!ssrOnly && services) {
668
739
  for (const module of services) {
669
740
  if (!fs.existsSync(`${rootClientPath}/services/${module}`))
670
741
  fs.mkdirSync(`${rootClientPath}/services/${module}`, { recursive: true });
@@ -727,7 +798,9 @@ const buildClient = async (
727
798
  const swSrcPath = `./src/client/sw/core.sw.js`;
728
799
  const swPublicPath = `${rootClientPath}/sw.js`;
729
800
  const swShouldRebuild =
730
- views && !(enableLiveRebuild && !options.liveClientBuildPaths.find((p) => p.srcBuildPath === swSrcPath));
801
+ !ssrOnly &&
802
+ views &&
803
+ !(enableLiveRebuild && !options.liveClientBuildPaths.find((p) => p.srcBuildPath === swSrcPath));
731
804
  // Transformed SW JS is held in memory; it gets prepended with renderPayload
732
805
  // and written once below, after PRE_CACHED_RESOURCES are known.
733
806
  let swTransformedJs = '';
@@ -741,7 +814,7 @@ const buildClient = async (
741
814
  });
742
815
  }
743
816
 
744
- if (views) {
817
+ if (!ssrOnly && views) {
745
818
  if (
746
819
  !(
747
820
  enableLiveRebuild &&
@@ -916,7 +989,7 @@ const buildClient = async (
916
989
  );
917
990
  }
918
991
  }
919
- if (!enableLiveRebuild && siteMapLinks.length > 0) {
992
+ if (!ssrOnly && !enableLiveRebuild && siteMapLinks.length > 0) {
920
993
  const hasSitemapTemplate = fs.existsSync(`${rootClientPath}/sitemap`);
921
994
  const sitemapBaseUrl = `https://${host}${path === '/' ? '' : path}`;
922
995
  // Create a stream to write to — omit xslUrl so we can inject a relative href below
@@ -953,7 +1026,7 @@ Sitemap: ${sitemapBaseUrl}/sitemap.xml`,
953
1026
  );
954
1027
  }
955
1028
 
956
- if (fullBuildEnabled && docs) {
1029
+ if (!ssrOnly && fullBuildEnabled && docs) {
957
1030
  await buildDocs({
958
1031
  host,
959
1032
  path,
@@ -977,6 +1050,8 @@ Sitemap: ${sitemapBaseUrl}/sitemap.xml`,
977
1050
  // when the network is unreachable.
978
1051
  const ssrClientConf = confSSR[getCapVariableName(client)] || {};
979
1052
  const ssrViews = Array.isArray(ssrClientConf.views) ? ssrClientConf.views : [];
1053
+ const statusPageRoutes = statusPageRoutesFactory({ views: ssrViews, proxyPath: path });
1054
+ if (statusPageRoutes.length > 0) logger.info('ssr status page routes', statusPageRoutes);
980
1055
  const PRE_CACHED_RESOURCES = [];
981
1056
  let offlineFallbackUrl = null;
982
1057
  let maintenanceFallbackUrl = null;
@@ -999,9 +1074,16 @@ Sitemap: ${sitemapBaseUrl}/sitemap.xml`,
999
1074
  renderApi: { JSONweb },
1000
1075
  });
1001
1076
 
1002
- const buildPath = `${
1003
- rootClientPath[rootClientPath.length - 1] === '/' ? rootClientPath.slice(0, -1) : rootClientPath
1004
- }${view.path === '/' ? view.path : `${view.path}/`}`;
1077
+ // A status view is built under `status-pages/<status>/`, not on its own
1078
+ // `/<status>` route: the gateway serves it by intercepting the
1079
+ // runtime's error, and a document on that route would give the runtime
1080
+ // a page of its own to serve or redirect to for the same condition.
1081
+ const statusCode = statusPageRoutes.find((route) => route.routePath === `${proxyPrefix}${view.path}`)?.status;
1082
+ const clientRoot =
1083
+ rootClientPath[rootClientPath.length - 1] === '/' ? rootClientPath.slice(0, -1) : rootClientPath;
1084
+ const buildPath = statusCode
1085
+ ? `${clientRoot}/${dir.dirname(statusPageBuildSegment(statusCode))}/`
1086
+ : `${clientRoot}${view.path === '/' ? view.path : `${view.path}/`}`;
1005
1087
 
1006
1088
  const indexUrl = buildIndexUrl(view.path);
1007
1089
  if (view.offlineDefault) {
@@ -1052,7 +1134,7 @@ ${swTransformedJs}`,
1052
1134
  );
1053
1135
  }
1054
1136
  }
1055
- if (!enableLiveRebuild && options.buildZip) {
1137
+ if (!ssrOnly && !enableLiveRebuild && options.buildZip) {
1056
1138
  logger.warn('build zip', rootClientPath);
1057
1139
 
1058
1140
  if (!fs.existsSync('./build')) fs.mkdirSync('./build');
@@ -1089,4 +1171,11 @@ ${swTransformedJs}`,
1089
1171
  }
1090
1172
  };
1091
1173
 
1092
- export { buildClient, copyNonExistingFiles, unzipClientBuild, mergeClientBuildZip };
1174
+ export {
1175
+ buildClient,
1176
+ copyNonExistingFiles,
1177
+ unzipClientBuild,
1178
+ mergeClientBuildZip,
1179
+ staticContextRoutesFactory,
1180
+ statusPageRoutesFactory,
1181
+ };
@@ -9,9 +9,8 @@ import vm from 'node:vm';
9
9
 
10
10
  import Underpost from '../index.js';
11
11
 
12
- import { srcFormatted, JSONweb } from './client-formatted.js';
12
+ import { srcFormatted } from './client-formatted.js';
13
13
  import { loggerFactory } from '../server/logger.js';
14
- import { getRootDirectory } from '../server/process.js';
15
14
 
16
15
  const logger = loggerFactory(import.meta);
17
16
 
@@ -47,78 +46,33 @@ const sanitizeHtml = (res, req, html) => {
47
46
  };
48
47
 
49
48
  /**
50
- * Factory function to create Express middleware for handling 404 and 500 errors.
51
- * It generates server-side rendered HTML for these error pages. If static error pages exist, it redirects to them.
52
- * @param {object} options - The options for creating the middleware.
53
- * @param {object} options.app - The Express app instance.
54
- * @param {string} options.directory - The directory for the instance's static files.
55
- * @param {string} options.rootHostPath - The root path for the host's public files.
56
- * @param {string} options.path - The base path for the instance.
57
- * @returns {Promise<{error500: Function, error400: Function}>} A promise that resolves to an object containing the 500 and 404 error handling middleware.
49
+ * Creates the Express middleware that terminates an unmatched request and an
50
+ * unhandled error.
51
+ *
52
+ * Both return a bare status and nothing else. Status page delivery belongs to
53
+ * the edge: the gateway intercepts the status and serves the declared document
54
+ * from `underpost-gateway`, preserving this response's code and the client's
55
+ * URI. A runtime that rendered its own page, redirected to one, or fetched one
56
+ * over HTTP would be competing with that and would be the only one of the
57
+ * three runtimes doing so.
58
+ * @param {string} [path] - The instance's proxy sub-path, used only to alias `/home`.
59
+ * @returns {Promise<{error500: Function, error400: Function}>} The two terminators.
58
60
  * @memberof ServerSideRendering
59
61
  */
60
- const ssrMiddlewareFactory = async ({ app, directory, rootHostPath, path }) => {
61
- const Render = await ssrFactory();
62
- const ssrPath = path === '/' ? path : `${path}/`;
63
-
64
- // Build default html src for 404 and 500
65
-
66
- const defaultHtmlSrc404 = Render({
67
- title: '404 Not Found',
68
- ssrPath,
69
- ssrHeadComponents: '',
70
- ssrBodyComponents: (await ssrFactory(`./src/client/ssr/body/404.js`))(),
71
- renderPayload: {
72
- apiBasePath: process.env.BASE_API,
73
- version: Underpost.version,
74
- },
75
- renderApi: {
76
- JSONweb,
77
- },
78
- });
79
- const path404 = `${directory ? directory : `${getRootDirectory()}${rootHostPath}`}/404/index.html`;
80
- const page404 = fs.existsSync(path404) ? `${path === '/' ? '' : path}/404` : undefined;
81
-
82
- const defaultHtmlSrc500 = Render({
83
- title: '500 Server Error',
84
- ssrPath,
85
- ssrHeadComponents: '',
86
- ssrBodyComponents: (await ssrFactory(`./src/client/ssr/body/500.js`))(),
87
- renderPayload: {
88
- apiBasePath: process.env.BASE_API,
89
- version: Underpost.version,
90
- },
91
- renderApi: {
92
- JSONweb,
93
- },
94
- });
95
- const path500 = `${directory ? directory : `${getRootDirectory()}${rootHostPath}`}/500/index.html`;
96
- const page500 = fs.existsSync(path500) ? `${path === '/' ? '' : path}/500` : undefined;
97
-
98
- return {
99
- error500: function (err, req, res, next) {
100
- logger.error(err, err.stack);
101
- if (page500) return res.status(500).redirect(page500);
102
- else {
103
- res.set('Content-Type', 'text/html');
104
- return res.status(500).send(sanitizeHtml(res, req, defaultHtmlSrc500));
105
- }
106
- },
107
- error400: function (req, res, next) {
108
- // if /<path>/home redirect to /<path>
109
- const homeRedirectPath = `${path === '/' ? '' : path}/home`;
110
- if (req.url.startsWith(homeRedirectPath)) {
111
- const redirectUrl = req.url.replace('/home', '');
112
- return res.redirect(redirectUrl.startsWith('/') ? redirectUrl : `/${redirectUrl}`);
113
- }
114
-
115
- if (page404) return res.status(404).redirect(page404);
116
- else {
117
- res.set('Content-Type', 'text/html');
118
- return res.status(404).send(sanitizeHtml(res, req, defaultHtmlSrc404));
119
- }
120
- },
121
- };
122
- };
62
+ const ssrMiddlewareFactory = async ({ path = '/' } = {}) => ({
63
+ error500: function (err, req, res, next) {
64
+ logger.error(err, err.stack);
65
+ return res.sendStatus(500);
66
+ },
67
+ error400: function (req, res, next) {
68
+ // `/<path>/home` is an alias of `/<path>`, not a missing route.
69
+ const homeRedirectPath = `${path === '/' ? '' : path}/home`;
70
+ if (req.url.startsWith(homeRedirectPath)) {
71
+ const redirectUrl = req.url.replace('/home', '');
72
+ return res.redirect(redirectUrl.startsWith('/') ? redirectUrl : `/${redirectUrl}`);
73
+ }
74
+ return res.sendStatus(404);
75
+ },
76
+ });
123
77
 
124
78
  export { ssrMiddlewareFactory, ssrFactory, sanitizeHtml };