underpost 3.2.30 → 3.2.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ghpkg.ci.yml +87 -0
- package/.github/workflows/npmpkg.ci.yml +9 -6
- package/.github/workflows/publish.ci.yml +3 -3
- package/.github/workflows/pwa-microservices-template-page.cd.yml +0 -7
- package/.github/workflows/release.cd.yml +1 -1
- package/CHANGELOG.md +1230 -971
- package/CLI-HELP.md +14 -6
- package/README.md +3 -3
- package/bin/build.js +40 -4
- package/bin/deploy.js +2 -2
- package/bump.config.js +1 -0
- package/docker-compose.yml +26 -26
- package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +1 -1
- package/manifests/cronjobs/dd-cron/dd-cron-dns.yaml +1 -1
- package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
- package/package.json +15 -15
- package/scripts/disk-clean.sh +85 -60
- package/scripts/test-monitor.sh +1 -1
- package/src/api/core/core.controller.js +4 -65
- package/src/api/core/core.router.js +8 -14
- package/src/api/default/default.controller.js +2 -70
- package/src/api/default/default.router.js +7 -17
- package/src/api/document/document.controller.js +5 -77
- package/src/api/document/document.router.js +9 -13
- package/src/api/file/file.controller.js +9 -53
- package/src/api/file/file.router.js +14 -6
- package/src/api/test/test.controller.js +8 -53
- package/src/api/test/test.router.js +1 -4
- package/src/cli/cluster.js +104 -11
- package/src/cli/db.js +4 -2
- package/src/cli/deploy.js +60 -11
- package/src/cli/docker-compose.js +212 -1
- package/src/cli/fs.js +45 -25
- package/src/cli/image.js +147 -51
- package/src/cli/index.js +26 -6
- package/src/cli/release.js +50 -4
- package/src/cli/repository.js +98 -24
- package/src/cli/run.js +380 -178
- package/src/cli/secrets.js +75 -46
- package/src/cli/ssh.js +30 -11
- package/src/client/components/core/Modal.js +38 -4
- package/src/index.js +1 -1
- package/src/server/catalog.js +2 -0
- package/src/server/conf.js +163 -3
- package/src/server/downloader.js +3 -3
- package/src/server/middlewares.js +152 -0
|
@@ -12,6 +12,7 @@ import fs from 'fs-extra';
|
|
|
12
12
|
import nodePath from 'path';
|
|
13
13
|
import { getRootDirectory, shellExec } from '../server/process.js';
|
|
14
14
|
import { loggerFactory } from '../server/logger.js';
|
|
15
|
+
import { loadInstanceTopology } from '../server/conf.js';
|
|
15
16
|
import Nginx from '../runtime/nginx/Nginx.js';
|
|
16
17
|
|
|
17
18
|
const logger = loggerFactory(import.meta);
|
|
@@ -77,14 +78,42 @@ class UnderpostDockerCompose {
|
|
|
77
78
|
return nodePath.isAbsolute(relPath) ? relPath : nodePath.join(getRootDirectory(), relPath);
|
|
78
79
|
}
|
|
79
80
|
|
|
81
|
+
/**
|
|
82
|
+
* Resolves the canonical directory for a custom docker-compose workflow,
|
|
83
|
+
* keyed by `--deploy-id` + `--docker-compose-id`:
|
|
84
|
+
* `engine-private/conf/<deploy-id>/docker-compose/<docker-compose-id>`.
|
|
85
|
+
* This directory ships its own `docker-compose.yml`, `compose.env`, and
|
|
86
|
+
* `nginx.conf` (used as-is, never generated). Returns null when
|
|
87
|
+
* `--docker-compose-id` is not set (the default, self-generating workflow).
|
|
88
|
+
* @param {object} options - CLI options.
|
|
89
|
+
* @returns {string|null} Repo-root-relative canonical dir, or null.
|
|
90
|
+
* @memberof UnderpostDockerCompose
|
|
91
|
+
*/
|
|
92
|
+
static composeIdBase(options = {}) {
|
|
93
|
+
if (!options.dockerComposeId) return null;
|
|
94
|
+
const deployId = options.deployId || DEFAULT_DEPLOY_ID;
|
|
95
|
+
return `engine-private/conf/${deployId}/docker-compose/${options.dockerComposeId}`;
|
|
96
|
+
}
|
|
97
|
+
|
|
80
98
|
/**
|
|
81
99
|
* Builds the base `docker compose` invocation with explicit file and env-file,
|
|
82
100
|
* so behavior is independent of the caller's working directory.
|
|
101
|
+
*
|
|
102
|
+
* Custom workflow (`--docker-compose-id`): the compose file, env-file, and
|
|
103
|
+
* bind-mounted config (nginx.conf, mongodb/) all live in the canonical dir, so
|
|
104
|
+
* compose runs with `--project-directory` pinned there and no app-override.
|
|
83
105
|
* @param {object} options - CLI options.
|
|
84
106
|
* @returns {string} The base command string (without a subcommand).
|
|
85
107
|
* @memberof UnderpostDockerCompose
|
|
86
108
|
*/
|
|
87
109
|
static baseCmd(options = {}) {
|
|
110
|
+
const base = UnderpostDockerCompose.composeIdBase(options);
|
|
111
|
+
if (base) {
|
|
112
|
+
const projectDir = UnderpostDockerCompose.resolve(base);
|
|
113
|
+
const composeFile = UnderpostDockerCompose.resolve(options.composeFile || `${base}/docker-compose.yml`);
|
|
114
|
+
const envFile = UnderpostDockerCompose.resolve(options.envFile || `${base}/compose.env`);
|
|
115
|
+
return `docker compose --project-directory ${projectDir} --env-file ${envFile} -f ${composeFile}`;
|
|
116
|
+
}
|
|
88
117
|
const composeFile = UnderpostDockerCompose.resolve(options.composeFile || 'docker-compose.yml');
|
|
89
118
|
const envFile = UnderpostDockerCompose.resolve(options.envFile || 'docker/compose.env');
|
|
90
119
|
const overrideFile = UnderpostDockerCompose.resolve(options.appOverride || 'docker/compose.app.yml');
|
|
@@ -163,7 +192,7 @@ services:
|
|
|
163
192
|
MONGO_IMAGE=mongo:latest
|
|
164
193
|
VALKEY_IMAGE=valkey/valkey:latest
|
|
165
194
|
APP_IMAGE=underpost/underpost-engine
|
|
166
|
-
APP_TAG=v3.2.
|
|
195
|
+
APP_TAG=v3.2.80
|
|
167
196
|
PROXY_IMAGE=nginx:stable-alpine
|
|
168
197
|
PROMETHEUS_IMAGE=prom/prometheus:latest
|
|
169
198
|
GRAFANA_IMAGE=grafana/grafana:latest
|
|
@@ -333,6 +362,150 @@ datasources:
|
|
|
333
362
|
`;
|
|
334
363
|
}
|
|
335
364
|
|
|
365
|
+
/**
|
|
366
|
+
* Expands the deploy's `multiInstance` topology (from conf.instances.json, via
|
|
367
|
+
* {@link loadInstanceTopology}) into the per-variant descriptors the compose
|
|
368
|
+
* generators consume. Application-agnostic: it only reads variant code/slug/
|
|
369
|
+
* path and the default code. Returns null when the deploy is single-instance.
|
|
370
|
+
* @param {object} options - CLI options.
|
|
371
|
+
* @returns {?{defaultCode: string, variants: Array<object>}}
|
|
372
|
+
* @memberof UnderpostDockerCompose
|
|
373
|
+
*/
|
|
374
|
+
static instanceTopology(options = {}) {
|
|
375
|
+
const deployId = options.deployId || DEFAULT_DEPLOY_ID;
|
|
376
|
+
let topology = null;
|
|
377
|
+
try {
|
|
378
|
+
topology = loadInstanceTopology(deployId);
|
|
379
|
+
} catch {
|
|
380
|
+
return null;
|
|
381
|
+
}
|
|
382
|
+
if (!topology || !Array.isArray(topology.variants) || topology.variants.length === 0) return null;
|
|
383
|
+
const defaultCode = topology.default || topology.variants[0].code;
|
|
384
|
+
const variants = topology.variants.map((v) => {
|
|
385
|
+
const path = v.path || '/';
|
|
386
|
+
const isDefault = !(v.slug || '');
|
|
387
|
+
return {
|
|
388
|
+
code: v.code,
|
|
389
|
+
slug: v.slug || '',
|
|
390
|
+
path,
|
|
391
|
+
isDefault,
|
|
392
|
+
// Service/container suffix: empty for the default variant so it keeps
|
|
393
|
+
// the historic `cyberia-server` / `cyberia-client` names (and their
|
|
394
|
+
// published host ports); `-forest`, `-test`, … for the rest.
|
|
395
|
+
suffix: v.slug ? `-${v.slug}` : '',
|
|
396
|
+
// Backend prefix strip, mirroring the K8s pathRewritePolicy: the Go
|
|
397
|
+
// server serves `/ws` at its root, so a `/FOREST` prefix is stripped.
|
|
398
|
+
strip: path !== '/',
|
|
399
|
+
};
|
|
400
|
+
});
|
|
401
|
+
return { defaultCode, variants };
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* Renders the nginx reverse-proxy config for the multi-instance Cyberia stack
|
|
406
|
+
* as a SINGLE localhost gateway (dev compose is single-host; production uses
|
|
407
|
+
* the K8s HTTPProxy, not this file). All three tiers share one origin, routed
|
|
408
|
+
* by URL sub-path so `http://localhost/` works with no /etc/hosts:
|
|
409
|
+
* /api/*, /assets/* -> engine-cyberia (shared content authority)
|
|
410
|
+
* /<slug>/ws -> cyberia-server-<slug> (prefix stripped; the Go
|
|
411
|
+
* runtime is instance-agnostic, selected by
|
|
412
|
+
* INSTANCE_CODE), and /ws -> the default server
|
|
413
|
+
* /<slug>, / -> cyberia-client-<slug> (prefix kept; the WASM
|
|
414
|
+
* reads the instance from the URL)
|
|
415
|
+
* Upstreams resolve lazily through Docker's embedded DNS so a variant that is
|
|
416
|
+
* briefly down never fails nginx startup. Returns null for single-instance.
|
|
417
|
+
* @param {object} options - CLI options.
|
|
418
|
+
* @returns {?string} nginx.conf content, or null.
|
|
419
|
+
* @memberof UnderpostDockerCompose
|
|
420
|
+
*/
|
|
421
|
+
static instancesNginxContent(options = {}) {
|
|
422
|
+
const topology = UnderpostDockerCompose.instanceTopology(options);
|
|
423
|
+
if (!topology) return null;
|
|
424
|
+
const { variants } = topology;
|
|
425
|
+
|
|
426
|
+
const headers = ` proxy_set_header Host $host;
|
|
427
|
+
proxy_set_header X-Real-IP $remote_addr;
|
|
428
|
+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
429
|
+
proxy_set_header X-Forwarded-Proto $scheme;
|
|
430
|
+
proxy_set_header Upgrade $http_upgrade;
|
|
431
|
+
proxy_set_header Connection $connection_upgrade;
|
|
432
|
+
proxy_read_timeout 3600s;`;
|
|
433
|
+
|
|
434
|
+
const proxyBlock = (variable, upstream, extra = '') =>
|
|
435
|
+
` set ${variable} ${upstream};
|
|
436
|
+
${extra} proxy_pass http://${variable};
|
|
437
|
+
${headers}`;
|
|
438
|
+
|
|
439
|
+
// Websocket tier: /<path>/ws -> server (prefix stripped); default -> = /ws.
|
|
440
|
+
// Longest-prefix ensures /<path>/ws beats the /<path> client location.
|
|
441
|
+
const wsLocations = variants
|
|
442
|
+
.map((v) => {
|
|
443
|
+
const upstream = `cyberia-server${v.suffix}:8081`;
|
|
444
|
+
const varName = `$up_server_${v.slug || 'default'}`;
|
|
445
|
+
if (v.isDefault) return ` location = /ws {\n${proxyBlock(varName, upstream)}\n }`;
|
|
446
|
+
return ` location ${v.path}/ws {\n${proxyBlock(
|
|
447
|
+
varName,
|
|
448
|
+
upstream,
|
|
449
|
+
` rewrite ^${v.path}/(.*)$ /$1 break;\n`,
|
|
450
|
+
)}\n }`;
|
|
451
|
+
})
|
|
452
|
+
.join('\n\n');
|
|
453
|
+
|
|
454
|
+
// Presentation tier: /<path> -> client (prefix kept); default -> catch-all /.
|
|
455
|
+
const clientLocations = variants
|
|
456
|
+
.map((v) => {
|
|
457
|
+
const upstream = `cyberia-client${v.suffix}:8081`;
|
|
458
|
+
const varName = `$up_client_${v.slug || 'default'}`;
|
|
459
|
+
if (v.isDefault) return ` location / {\n${proxyBlock(varName, upstream)}\n }`;
|
|
460
|
+
return ` location = ${v.path} { return 301 ${v.path}/; }
|
|
461
|
+
location ${v.path} {\n${proxyBlock(varName, upstream)}\n }`;
|
|
462
|
+
})
|
|
463
|
+
.join('\n\n');
|
|
464
|
+
|
|
465
|
+
const deployId = options.deployId || DEFAULT_DEPLOY_ID;
|
|
466
|
+
return `# Generated by 'underpost docker-compose --generate --deploy-id ${deployId} --docker-compose-id ${options.dockerComposeId || 'cyberia'}' — do not hand-edit.
|
|
467
|
+
# Source of truth: engine-private/conf/${deployId}/conf.instances.json (multiInstance).
|
|
468
|
+
# Single localhost gateway (dev compose). Access http://localhost/ (default),
|
|
469
|
+
# http://localhost/<PATH> per variant. Client origins point back here
|
|
470
|
+
# (compose.env CYBERIA_WS_ORIGIN=ws://localhost, CYBERIA_ENGINE_API_ORIGIN=http://localhost).
|
|
471
|
+
|
|
472
|
+
map $http_upgrade $connection_upgrade {
|
|
473
|
+
default upgrade;
|
|
474
|
+
'' close;
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
# Docker embedded DNS: resolve upstream names per request so a variant that is
|
|
478
|
+
# briefly down never fails nginx startup or config reload.
|
|
479
|
+
resolver 127.0.0.11 ipv6=off valid=10s;
|
|
480
|
+
|
|
481
|
+
proxy_http_version 1.1;
|
|
482
|
+
|
|
483
|
+
server {
|
|
484
|
+
listen 80 default_server;
|
|
485
|
+
server_name localhost 127.0.0.1 _;
|
|
486
|
+
|
|
487
|
+
location = /healthz {
|
|
488
|
+
access_log off;
|
|
489
|
+
return 200 "ok\\n";
|
|
490
|
+
add_header Content-Type text/plain;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
# Shared content authority: REST API + engine-served assets (fonts, ui-icons).
|
|
494
|
+
location /api/ {
|
|
495
|
+
${proxyBlock('$up_engine', 'engine-cyberia:4005')}
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
location /assets/ {
|
|
499
|
+
${proxyBlock('$up_engine', 'engine-cyberia:4005')}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
${wsLocations}
|
|
503
|
+
|
|
504
|
+
${clientLocations}
|
|
505
|
+
}
|
|
506
|
+
`;
|
|
507
|
+
}
|
|
508
|
+
|
|
336
509
|
/**
|
|
337
510
|
* Renders all dynamic supporting files: the nginx reverse-proxy config (from
|
|
338
511
|
* PROXY_HOSTS), the monitoring configs (Prometheus + Grafana datasource), and
|
|
@@ -343,6 +516,28 @@ datasources:
|
|
|
343
516
|
* @memberof UnderpostDockerCompose
|
|
344
517
|
*/
|
|
345
518
|
static generate(options = {}) {
|
|
519
|
+
// Custom workflow: docker-compose.yml and compose.env are hand-authored in
|
|
520
|
+
// the canonical dir and used as-is — do NOT generate them. Only (re)write
|
|
521
|
+
// the two generated artifacts: the MongoDB entrypoint (replica-set
|
|
522
|
+
// bootstrap) and, for a multi-instance deploy, nginx.conf (per-variant
|
|
523
|
+
// sub-path routing derived from the conf.instances.json multiInstance block,
|
|
524
|
+
// kept in sync with the service tiers hand-authored in docker-compose.yml).
|
|
525
|
+
const composeIdBase = UnderpostDockerCompose.composeIdBase(options);
|
|
526
|
+
if (composeIdBase) {
|
|
527
|
+
const mongoEntrypointPath = UnderpostDockerCompose.resolve(`${composeIdBase}/mongodb/entrypoint.sh`);
|
|
528
|
+
fs.mkdirpSync(nodePath.dirname(mongoEntrypointPath));
|
|
529
|
+
fs.writeFileSync(mongoEntrypointPath, UnderpostDockerCompose.mongoEntrypointContent(), { mode: 0o755 });
|
|
530
|
+
logger.info('mongodb entrypoint written (custom workflow)', { path: mongoEntrypointPath });
|
|
531
|
+
|
|
532
|
+
const nginx = UnderpostDockerCompose.instancesNginxContent(options);
|
|
533
|
+
if (nginx) {
|
|
534
|
+
const nginxPath = UnderpostDockerCompose.resolve(`${composeIdBase}/nginx.conf`);
|
|
535
|
+
fs.writeFileSync(nginxPath, nginx, 'utf8');
|
|
536
|
+
logger.info('multi-instance nginx.conf written', { path: nginxPath });
|
|
537
|
+
}
|
|
538
|
+
return;
|
|
539
|
+
}
|
|
540
|
+
|
|
346
541
|
Nginx.removeRouter();
|
|
347
542
|
for (const { host, routes } of PROXY_HOSTS) Nginx.createApp({ host, routes });
|
|
348
543
|
Nginx.createDefaultServer(DEFAULT_UPSTREAM);
|
|
@@ -449,6 +644,21 @@ datasources:
|
|
|
449
644
|
silentOnError: true,
|
|
450
645
|
});
|
|
451
646
|
|
|
647
|
+
// Custom workflow: docker-compose.yml and compose.env are hand-authored
|
|
648
|
+
// source — never prune them. Drop only the generated mongo entrypoint;
|
|
649
|
+
// nginx.conf is left in place so the mount stays valid and --generate/--up
|
|
650
|
+
// rewrites it.
|
|
651
|
+
const composeIdBase = UnderpostDockerCompose.composeIdBase(options);
|
|
652
|
+
if (composeIdBase) {
|
|
653
|
+
const mongoEntrypointPath = UnderpostDockerCompose.resolve(`${composeIdBase}/mongodb/entrypoint.sh`);
|
|
654
|
+
if (fs.existsSync(mongoEntrypointPath)) {
|
|
655
|
+
fs.removeSync(mongoEntrypointPath);
|
|
656
|
+
logger.info('removed generated artifact', { path: mongoEntrypointPath });
|
|
657
|
+
}
|
|
658
|
+
logger.info('Docker Compose reset complete. Run `--up` to recreate the stack.');
|
|
659
|
+
return;
|
|
660
|
+
}
|
|
661
|
+
|
|
452
662
|
// Phase 2: prune generated artifacts (regenerated on the next --generate/--up).
|
|
453
663
|
const artifacts = options.force
|
|
454
664
|
? [...GENERATED_ARTIFACTS, options.envFile || 'docker/compose.env']
|
|
@@ -489,6 +699,7 @@ datasources:
|
|
|
489
699
|
* @param {boolean} [options.shell] - Open an interactive shell in `target` (default: app).
|
|
490
700
|
* @param {string} [options.exec] - General-purpose passthrough docker compose subcommand.
|
|
491
701
|
* @param {string} [options.deployId] - Deployment to run as the app (default: dd-default). `dd-default` self-bootstraps a fresh engine; any other id runs the standard `underpost start` command.
|
|
702
|
+
* @param {string} [options.dockerComposeId] - Custom-workflow selector. When set, use the canonical stack at `engine-private/conf/<deploy-id>/docker-compose/<docker-compose-id>/` (docker-compose.yml + compose.env + nginx.conf, used as-is), skipping nginx/env/app-override generation. Used by the Cyberia MMO ecosystem (`--deploy-id dd-cyberia --docker-compose-id cyberia`).
|
|
492
703
|
* @param {string} [options.env] - Deployment environment for non-default deploy ids (default: development).
|
|
493
704
|
* @param {string} [options.composeFile] - Override compose file path.
|
|
494
705
|
* @param {string} [options.envFile] - Override env-file path.
|
package/src/cli/fs.js
CHANGED
|
@@ -64,6 +64,31 @@ class UnderpostFileStorage {
|
|
|
64
64
|
writeStorageConf(storage, storageConf) {
|
|
65
65
|
if (storage) fs.writeFileSync(storageConf, JSON.stringify(storage, null, 4), 'utf8');
|
|
66
66
|
},
|
|
67
|
+
/**
|
|
68
|
+
* @method gitTrack
|
|
69
|
+
* @description Optional, non-fatal Git tracking layer. Any Git error is logged and swallowed
|
|
70
|
+
* so it can never interrupt or roll back the canonical `storage.*.json` workflow.
|
|
71
|
+
* @param {string} gitPath - The working directory to stage/commit.
|
|
72
|
+
* @param {object} [options] - Tracking options.
|
|
73
|
+
* @param {boolean} [options.init=false] - If true, initialize a local repo before staging.
|
|
74
|
+
* @param {string} [options.message=''] - Explicit commit message; when omitted, `underpost cmt` is used.
|
|
75
|
+
* @memberof UnderpostFileStorage
|
|
76
|
+
*/
|
|
77
|
+
gitTrack(gitPath, options = { init: false, message: '' }) {
|
|
78
|
+
try {
|
|
79
|
+
if (options.init === true) Underpost.repo.initLocalRepo({ path: gitPath });
|
|
80
|
+
shellExec(`cd ${gitPath} && git add .`, { silentOnError: true, silent: true, disableLog: true });
|
|
81
|
+
if (options.message)
|
|
82
|
+
shellExec(`cd ${gitPath} && git commit -m "${options.message}"`, {
|
|
83
|
+
silentOnError: true,
|
|
84
|
+
silent: true,
|
|
85
|
+
disableLog: true,
|
|
86
|
+
});
|
|
87
|
+
else shellExec(`underpost cmt ${gitPath} feat`, { silentOnError: true, silent: true, disableLog: true });
|
|
88
|
+
} catch (error) {
|
|
89
|
+
logger.warn('git tracking skipped (non-fatal)', { gitPath, error: error?.message });
|
|
90
|
+
}
|
|
91
|
+
},
|
|
67
92
|
/**
|
|
68
93
|
* @method recursiveCallback
|
|
69
94
|
* @description Recursively processes files and directories based on the provided options.
|
|
@@ -125,6 +150,7 @@ class UnderpostFileStorage {
|
|
|
125
150
|
|
|
126
151
|
if (hasPathFilter && options.force === true && fs.existsSync(basePath)) fs.removeSync(basePath);
|
|
127
152
|
|
|
153
|
+
// Storage is canonical: persist the removal before any (optional) git tracking runs.
|
|
128
154
|
Underpost.fs.writeStorageConf(storage, storageConf);
|
|
129
155
|
|
|
130
156
|
if (associatedPaths.length === 0)
|
|
@@ -136,13 +162,8 @@ class UnderpostFileStorage {
|
|
|
136
162
|
});
|
|
137
163
|
|
|
138
164
|
if (options.git === true) {
|
|
139
|
-
const gitPath = hasPathFilter ?
|
|
140
|
-
|
|
141
|
-
shellExec(`underpost cmt ${gitPath} feat`, {
|
|
142
|
-
silentOnError: true,
|
|
143
|
-
silent: true,
|
|
144
|
-
disableLog: true,
|
|
145
|
-
});
|
|
165
|
+
const gitPath = !hasPathFilter ? '.' : isSingleFile ? parentDir : basePath;
|
|
166
|
+
Underpost.fs.gitTrack(gitPath);
|
|
146
167
|
}
|
|
147
168
|
|
|
148
169
|
return;
|
|
@@ -151,7 +172,16 @@ class UnderpostFileStorage {
|
|
|
151
172
|
// For single files, run getDeleteFiles against the parent directory to avoid
|
|
152
173
|
// trying to `cd` into a file.
|
|
153
174
|
const gitContextPath = isSingleFile ? parentDir : path;
|
|
154
|
-
|
|
175
|
+
// Detecting locally-deleted files is a best-effort enhancement backed by git; if the path is
|
|
176
|
+
// not a repo (or git is unavailable) it must not block the canonical storage workflow.
|
|
177
|
+
let deleteFiles = [];
|
|
178
|
+
if (options.pull !== true) {
|
|
179
|
+
try {
|
|
180
|
+
deleteFiles = Underpost.repo.getDeleteFiles(gitContextPath);
|
|
181
|
+
} catch (error) {
|
|
182
|
+
logger.warn('delete detection skipped (git unavailable)', { path: gitContextPath, error: error?.message });
|
|
183
|
+
}
|
|
184
|
+
}
|
|
155
185
|
|
|
156
186
|
// When processing a single file, only consider it for deletion
|
|
157
187
|
for (const relativePath of deleteFiles) {
|
|
@@ -172,12 +202,7 @@ class UnderpostFileStorage {
|
|
|
172
202
|
if (pullSkipCount > 0) logger.warn(`Pull skipped ${pullSkipCount} files that already exist`);
|
|
173
203
|
// Only run git init/commit when the caller explicitly requests git tracking (--git flag).
|
|
174
204
|
// For bundle pulls into ./build the git step is unwanted and would error on a non-repo path.
|
|
175
|
-
if (options.git === true) {
|
|
176
|
-
Underpost.repo.initLocalRepo({ path: gitContextPath });
|
|
177
|
-
shellExec(`cd ${gitContextPath} && git add . && git commit -m "Base pull state"`, {
|
|
178
|
-
silentOnError: true,
|
|
179
|
-
});
|
|
180
|
-
}
|
|
205
|
+
if (options.git === true) Underpost.fs.gitTrack(gitContextPath, { init: true, message: 'Base pull state' });
|
|
181
206
|
} else {
|
|
182
207
|
let files;
|
|
183
208
|
if (isSingleFile) {
|
|
@@ -200,15 +225,9 @@ class UnderpostFileStorage {
|
|
|
200
225
|
} else logger.warn('File already exists', _path);
|
|
201
226
|
}
|
|
202
227
|
}
|
|
228
|
+
// Storage is canonical and always persisted; git is an optional layer on top.
|
|
203
229
|
Underpost.fs.writeStorageConf(storage, storageConf);
|
|
204
|
-
if (options.git === true)
|
|
205
|
-
shellExec(`cd ${gitContextPath} && git add .`);
|
|
206
|
-
shellExec(`underpost cmt ${gitContextPath} feat`, {
|
|
207
|
-
silentOnError: true,
|
|
208
|
-
silent: true,
|
|
209
|
-
disableLog: true,
|
|
210
|
-
});
|
|
211
|
-
}
|
|
230
|
+
if (options.git === true) Underpost.fs.gitTrack(gitContextPath);
|
|
212
231
|
},
|
|
213
232
|
/**
|
|
214
233
|
* @method callback
|
|
@@ -230,10 +249,12 @@ class UnderpostFileStorage {
|
|
|
230
249
|
path,
|
|
231
250
|
options = { rm: false, recursive: false, deployId: '', force: false, pull: false, git: false, omitUnzip: false },
|
|
232
251
|
) {
|
|
233
|
-
|
|
252
|
+
// rm always routes through recursiveCallback so storage.*.json is updated regardless of
|
|
253
|
+
// --recursive/--git. The bare `delete` primitive only removes the remote asset and would
|
|
254
|
+
// otherwise leave the tracked storage key orphaned.
|
|
255
|
+
if (options.recursive === true || options.git === true || options.rm === true)
|
|
234
256
|
return await Underpost.fs.recursiveCallback(path, options);
|
|
235
257
|
if (options.pull === true) return await Underpost.fs.pull(path, options);
|
|
236
|
-
if (options.rm === true) return await Underpost.fs.delete(path, options);
|
|
237
258
|
return await Underpost.fs.upload(path, options);
|
|
238
259
|
},
|
|
239
260
|
/**
|
|
@@ -294,7 +315,6 @@ class UnderpostFileStorage {
|
|
|
294
315
|
public_ids: [path],
|
|
295
316
|
resource_type: 'raw',
|
|
296
317
|
});
|
|
297
|
-
logger.info('download result', downloadResult);
|
|
298
318
|
await Downloader.downloadFile(downloadResult, zipPath);
|
|
299
319
|
|
|
300
320
|
if (options.omitUnzip === true) {
|
package/src/cli/image.js
CHANGED
|
@@ -5,9 +5,12 @@
|
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
7
|
import fs from 'fs-extra';
|
|
8
|
+
import os from 'os';
|
|
9
|
+
import nodePath from 'path';
|
|
10
|
+
import crypto from 'crypto';
|
|
8
11
|
import { loggerFactory } from '../server/logger.js';
|
|
9
12
|
import Underpost from '../index.js';
|
|
10
|
-
import { getNpmRootPath
|
|
13
|
+
import { getNpmRootPath } from '../server/conf.js';
|
|
11
14
|
import { shellExec } from '../server/process.js';
|
|
12
15
|
|
|
13
16
|
const logger = loggerFactory(import.meta);
|
|
@@ -23,59 +26,32 @@ class UnderpostImage {
|
|
|
23
26
|
static API = {
|
|
24
27
|
/**
|
|
25
28
|
* @method pullBaseImages
|
|
26
|
-
* @description
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
* @param {boolean} [options.k3s=false] - If true, load image into K3s cluster.
|
|
32
|
-
* @param {string} [options.path=''] - Path to the Dockerfile context.
|
|
33
|
-
* @param {boolean} [options.dev=false] - If true, use development mode.
|
|
34
|
-
* @param {string} [options.version=''] - Version tag for the image.
|
|
35
|
-
* @param {string} [options.imageName=''] - Custom name for the image.
|
|
29
|
+
* @description Ensures the base image prerequisites for the runtime Dockerfiles
|
|
30
|
+
* are present on the host (currently `docker.io/rockylinux/rockylinux:9`). This
|
|
31
|
+
* only pulls — it does NOT build. Builds run with `podman build --pull=never`,
|
|
32
|
+
* so the base must exist locally beforehand; that is the sole purpose of this
|
|
33
|
+
* step. Combine with `--build` in the same command to pull-then-build.
|
|
36
34
|
* @memberof UnderpostImage
|
|
37
35
|
*/
|
|
38
|
-
pullBaseImages(
|
|
39
|
-
|
|
40
|
-
kind: false,
|
|
41
|
-
kubeadm: false,
|
|
42
|
-
k3s: false,
|
|
43
|
-
path: '',
|
|
44
|
-
dev: false,
|
|
45
|
-
version: '',
|
|
46
|
-
imageName: '',
|
|
47
|
-
},
|
|
48
|
-
) {
|
|
49
|
-
shellExec(`sudo podman pull docker.io/library/rockylinux:9`);
|
|
50
|
-
const baseCommand = options.dev ? 'node bin' : 'underpost';
|
|
51
|
-
const baseCommandOption = options.dev ? ' --dev' : '';
|
|
52
|
-
const IMAGE_NAME = options.imageName
|
|
53
|
-
? `options.imageName${options.version ? `:${options.version}` : ''}`
|
|
54
|
-
: `rockylinux9-underpost:${options.version ? options.version : Underpost.version}`;
|
|
55
|
-
let LOAD_TYPE = '';
|
|
56
|
-
if (options.kind === true) LOAD_TYPE = `--kind`;
|
|
57
|
-
else if (options.kubeadm === true) LOAD_TYPE = `--kubeadm`;
|
|
58
|
-
else if (options.k3s === true) LOAD_TYPE = `--k3s`;
|
|
59
|
-
shellExec(
|
|
60
|
-
`${baseCommand} image${baseCommandOption} --build --podman-save --reset --image-path=. --path ${
|
|
61
|
-
options.path ? options.path : getUnderpostRootPath()
|
|
62
|
-
} --image-name=${IMAGE_NAME} ${LOAD_TYPE}`,
|
|
63
|
-
);
|
|
36
|
+
pullBaseImages() {
|
|
37
|
+
shellExec(`sudo podman pull docker.io/rockylinux/rockylinux:9`);
|
|
64
38
|
},
|
|
65
39
|
/**
|
|
66
40
|
* @method build
|
|
67
41
|
* @description Builds a Docker image using Podman, optionally saves it as a tar archive,
|
|
68
|
-
* and loads it into a specified
|
|
42
|
+
* and loads it into a specified target (Kind, Kubeadm, or K3s cluster, or the local
|
|
43
|
+
* Docker store for Docker Compose).
|
|
69
44
|
* @param {object} options - Options for building and loading images.
|
|
70
45
|
* @param {string} [options.path=''] - The path to the directory containing the Dockerfile.
|
|
71
46
|
* @param {string} [options.imageName=''] - The name and tag for the image (e.g., 'my-app:latest').
|
|
72
47
|
* @param {string} [options.version=''] - Version tag for the image.
|
|
73
|
-
* @param {string} [options.
|
|
48
|
+
* @param {string} [options.imageOutPath=''] - Directory to save the image tar file.
|
|
74
49
|
* @param {string} [options.dockerfileName=''] - Name of the Dockerfile (defaults to 'Dockerfile').
|
|
75
50
|
* @param {boolean} [options.podmanSave=false] - If true, save the image as a tar archive using Podman.
|
|
76
51
|
* @param {boolean} [options.kind=false] - If true, load the image archive into a Kind cluster.
|
|
77
52
|
* @param {boolean} [options.kubeadm=false] - If true, load the image archive into a Kubeadm cluster (uses 'ctr').
|
|
78
53
|
* @param {boolean} [options.k3s=false] - If true, load the image archive into a K3s cluster (uses 'k3s ctr').
|
|
54
|
+
* @param {boolean} [options.dockerCompose=false] - If true, load the image archive into the local Docker store for Docker Compose.
|
|
79
55
|
* @param {boolean} [options.reset=false] - If true, perform a no-cache build.
|
|
80
56
|
* @param {boolean} [options.dev=false] - If true, use development mode.
|
|
81
57
|
* @memberof UnderpostImage
|
|
@@ -85,42 +61,162 @@ class UnderpostImage {
|
|
|
85
61
|
path: '',
|
|
86
62
|
imageName: '',
|
|
87
63
|
version: '',
|
|
88
|
-
|
|
64
|
+
imageOutPath: '',
|
|
89
65
|
dockerfileName: '',
|
|
90
66
|
podmanSave: false,
|
|
91
67
|
kind: false,
|
|
92
68
|
kubeadm: false,
|
|
93
69
|
k3s: false,
|
|
70
|
+
dockerCompose: false,
|
|
94
71
|
reset: false,
|
|
95
72
|
dev: false,
|
|
96
73
|
},
|
|
97
74
|
) {
|
|
98
|
-
let {
|
|
75
|
+
let {
|
|
76
|
+
path,
|
|
77
|
+
imageName,
|
|
78
|
+
version,
|
|
79
|
+
imageOutPath,
|
|
80
|
+
dockerfileName,
|
|
81
|
+
podmanSave,
|
|
82
|
+
kind,
|
|
83
|
+
kubeadm,
|
|
84
|
+
k3s,
|
|
85
|
+
dockerCompose,
|
|
86
|
+
reset,
|
|
87
|
+
dev,
|
|
88
|
+
} = options;
|
|
99
89
|
if (!path) path = '.';
|
|
100
90
|
if (!imageName) imageName = `rockylinux9-underpost:${Underpost.version}`;
|
|
101
|
-
if (!
|
|
91
|
+
if (!imageOutPath) imageOutPath = '.';
|
|
102
92
|
if (imageName.match('/')) imageName = imageName.split('/')[1];
|
|
103
93
|
if (!version) version = 'latest';
|
|
104
94
|
version = imageName && imageName.match(':') ? '' : `:${version}`;
|
|
105
95
|
const podManImg = `localhost/${imageName}${version}`;
|
|
106
|
-
if (
|
|
107
|
-
fs.mkdirSync(
|
|
108
|
-
const tarFile = `${
|
|
96
|
+
if (imageOutPath && typeof imageOutPath === 'string' && !fs.existsSync(imageOutPath))
|
|
97
|
+
fs.mkdirSync(imageOutPath, { recursive: true });
|
|
98
|
+
const tarFile = `${imageOutPath}/${imageName.replace(':', '_')}.tar`;
|
|
109
99
|
let cache = '';
|
|
110
100
|
if (reset === true) cache += ' --rm --no-cache';
|
|
101
|
+
|
|
102
|
+
// Forward GitHub credentials from the host environment into the build as
|
|
103
|
+
// podman/BuildKit secrets (`--secret id=...,src=...`), matching the
|
|
104
|
+
// `RUN --mount=type=secret,id=github_*` contract in the runtime
|
|
105
|
+
// Dockerfiles (e.g. src/runtime/engine-cyberia). Secrets are written to
|
|
106
|
+
// short-lived 0600 temp files and removed right after the build — they are
|
|
107
|
+
// never passed as build-args (which would persist in image history) nor
|
|
108
|
+
// baked into any layer.
|
|
109
|
+
const secretTmpFiles = [];
|
|
110
|
+
const secretFlags = [];
|
|
111
|
+
const addBuildSecret = (id, value) => {
|
|
112
|
+
if (!value) return;
|
|
113
|
+
const file = nodePath.join(os.tmpdir(), `underpost-secret-${id}-${crypto.randomBytes(6).toString('hex')}`);
|
|
114
|
+
fs.writeFileSync(file, String(value), { mode: 0o600 });
|
|
115
|
+
secretTmpFiles.push(file);
|
|
116
|
+
secretFlags.push(`--secret id=${id},src=${file}`);
|
|
117
|
+
};
|
|
118
|
+
// addBuildSecret('github_token', process.env.GITHUB_TOKEN);
|
|
119
|
+
addBuildSecret('github_username', process.env.GITHUB_USERNAME);
|
|
120
|
+
// Build secrets are created dynamically: each `RUN --mount=type=secret,id=<x>`
|
|
121
|
+
// id maps to a host env var; only the ones actually set are passed, so a
|
|
122
|
+
// Dockerfile that omits (or comments out) a mount just never sees it. Extend
|
|
123
|
+
// this map — plus a merge of any caller-supplied `options.buildSecrets` — to
|
|
124
|
+
// add more without touching the call sites. Secrets never persist in image
|
|
125
|
+
// history (unlike build-args); they live in 0600 temp files removed post-build.
|
|
126
|
+
const BUILD_SECRET_ENV = {
|
|
127
|
+
// // Cloudinary creds power build-time asset pulls (`node bin fs --pull`).
|
|
128
|
+
// cloudinary_cloud_name: 'CLOUDINARY_CLOUD_NAME',
|
|
129
|
+
// cloudinary_api_key: 'CLOUDINARY_API_KEY',
|
|
130
|
+
// cloudinary_api_secret: 'CLOUDINARY_API_SECRET',
|
|
131
|
+
};
|
|
132
|
+
for (const [id, envVar] of Object.entries(BUILD_SECRET_ENV)) addBuildSecret(id, process.env[envVar]);
|
|
133
|
+
for (const [id, value] of Object.entries(options.buildSecrets || {})) addBuildSecret(id, value);
|
|
134
|
+
const secretArgs = secretFlags.length ? ` ${secretFlags.join(' ')}` : '';
|
|
135
|
+
if (secretFlags.length) logger.info('Passing host credentials as build secrets', { ids: secretFlags.length });
|
|
136
|
+
|
|
137
|
+
// Non-secret build args (e.g. INSTANCE_CODES for engine-cyberia): injected
|
|
138
|
+
// dynamically from `options.buildArgs`, overriding the Dockerfile's ARG
|
|
139
|
+
// defaults. Unlike secrets these DO persist in image metadata, so only
|
|
140
|
+
// non-sensitive values belong here.
|
|
141
|
+
const buildArgFlags = Object.entries(options.buildArgs || {})
|
|
142
|
+
.filter(([, v]) => v !== undefined && v !== null && v !== '')
|
|
143
|
+
.map(([k, v]) => `--build-arg ${k}=${JSON.stringify(String(v))}`);
|
|
144
|
+
const buildArgStr = buildArgFlags.length ? ` ${buildArgFlags.join(' ')}` : '';
|
|
145
|
+
|
|
111
146
|
if (path)
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
147
|
+
try {
|
|
148
|
+
shellExec(
|
|
149
|
+
`cd ${path} && sudo podman build -f ./${
|
|
150
|
+
dockerfileName && typeof dockerfileName === 'string' ? dockerfileName : 'Dockerfile'
|
|
151
|
+
} -t ${imageName} --pull=never --cap-add=CAP_AUDIT_WRITE${cache}${secretArgs}${buildArgStr} --network host`,
|
|
152
|
+
);
|
|
153
|
+
} finally {
|
|
154
|
+
for (const file of secretTmpFiles) {
|
|
155
|
+
try {
|
|
156
|
+
fs.removeSync(file);
|
|
157
|
+
} catch {
|
|
158
|
+
/* best-effort cleanup */
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
// Loading into any target requires the tar archive, so imply the save when
|
|
163
|
+
// one is set (kind/kubeadm/k3s/docker-compose) even if --podman-save was omitted.
|
|
164
|
+
const loadTarget = kind === true || kubeadm === true || k3s === true || dockerCompose === true;
|
|
165
|
+
if (podmanSave === true || loadTarget) {
|
|
118
166
|
if (fs.existsSync(tarFile)) fs.removeSync(tarFile);
|
|
119
167
|
shellExec(`podman save -o ${tarFile} ${podManImg}`);
|
|
120
168
|
}
|
|
121
169
|
if (kind === true) shellExec(`sudo kind load image-archive ${tarFile}`);
|
|
122
170
|
else if (kubeadm === true) shellExec(`sudo ctr -n k8s.io images import ${tarFile}`);
|
|
123
171
|
else if (k3s === true) shellExec(`sudo k3s ctr images import ${tarFile}`);
|
|
172
|
+
// Independent of any cluster target: make the local image available to the
|
|
173
|
+
// Docker daemon so `docker compose` can resolve it (e.g. ENGINE_CYBERIA_IMAGE).
|
|
174
|
+
if (dockerCompose === true) shellExec(`sudo docker load -i ${tarFile}`);
|
|
175
|
+
},
|
|
176
|
+
/**
|
|
177
|
+
* @method importTar
|
|
178
|
+
* @description Loads a pre-built image tar archive into each enabled target
|
|
179
|
+
* without building anything. Mirrors the load step of {@link build}, but
|
|
180
|
+
* the archive is supplied directly via `--import-tar <tar-path>` and every
|
|
181
|
+
* enabled target flag is honored (the same archive is loaded into each), so
|
|
182
|
+
* `--kind --docker-compose` loads it into both.
|
|
183
|
+
* @param {object} options - CLI options.
|
|
184
|
+
* @param {string} options.importTar - Path to the image tar archive (e.g. `./image-v1.0.0.tar`).
|
|
185
|
+
* @param {boolean} [options.kind] - Load into the Kind cluster (`kind load image-archive`).
|
|
186
|
+
* @param {boolean} [options.kubeadm] - Import into kubeadm containerd (`ctr -n k8s.io images import`).
|
|
187
|
+
* @param {boolean} [options.k3s] - Import into k3s containerd (`k3s ctr images import`).
|
|
188
|
+
* @param {boolean} [options.dockerCompose] - Load into the local Docker daemon (`docker load`) for Docker Compose.
|
|
189
|
+
* @returns {void}
|
|
190
|
+
* @memberof UnderpostImage
|
|
191
|
+
*/
|
|
192
|
+
importTar(options = { importTar: '', kind: false, kubeadm: false, k3s: false, dockerCompose: false }) {
|
|
193
|
+
const { importTar, kind, kubeadm, k3s, dockerCompose } = options;
|
|
194
|
+
if (!importTar || typeof importTar !== 'string' || !fs.existsSync(importTar)) {
|
|
195
|
+
logger.error('image --import-tar: archive not found', { importTar });
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
const targets = [];
|
|
199
|
+
if (kind === true) {
|
|
200
|
+
shellExec(`sudo kind load image-archive ${importTar}`);
|
|
201
|
+
targets.push('kind');
|
|
202
|
+
}
|
|
203
|
+
if (kubeadm === true) {
|
|
204
|
+
shellExec(`sudo ctr -n k8s.io images import ${importTar}`);
|
|
205
|
+
targets.push('kubeadm');
|
|
206
|
+
}
|
|
207
|
+
if (k3s === true) {
|
|
208
|
+
shellExec(`sudo k3s ctr images import ${importTar}`);
|
|
209
|
+
targets.push('k3s');
|
|
210
|
+
}
|
|
211
|
+
if (dockerCompose === true) {
|
|
212
|
+
shellExec(`sudo docker load -i ${importTar}`);
|
|
213
|
+
targets.push('docker-compose');
|
|
214
|
+
}
|
|
215
|
+
if (targets.length === 0)
|
|
216
|
+
logger.warn(
|
|
217
|
+
'image --import-tar: no target enabled; combine with --kind, --kubeadm, --k3s and/or --docker-compose',
|
|
218
|
+
);
|
|
219
|
+
else logger.info('image --import-tar: archive loaded', { importTar, targets });
|
|
124
220
|
},
|
|
125
221
|
/**
|
|
126
222
|
* @method getCurrentLoaded
|