underpost 3.2.10 → 3.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.vscode/extensions.json +9 -9
- package/.vscode/settings.json +12 -1
- package/CHANGELOG.md +74 -1
- package/CLI-HELP.md +80 -26
- package/README.md +3 -3
- package/bin/build.js +9 -6
- package/bin/build.template.js +187 -0
- package/bin/deploy.js +29 -18
- package/conf.js +1 -4
- package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +1 -1
- package/manifests/cronjobs/dd-cron/dd-cron-dns.yaml +1 -1
- package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
- package/manifests/deployment/dd-test-development/deployment.yaml +2 -2
- package/manifests/lxd/lxd-admin-profile.yaml +12 -3
- package/manifests/mongodb-4.4/headless-service.yaml +10 -0
- package/manifests/mongodb-4.4/kustomization.yaml +3 -1
- package/manifests/mongodb-4.4/mongodb-nodeport.yaml +17 -0
- package/manifests/mongodb-4.4/pv-pvc.yaml +10 -14
- package/manifests/mongodb-4.4/statefulset.yaml +79 -0
- package/manifests/mongodb-4.4/storage-class.yaml +9 -0
- package/manifests/valkey/statefulset.yaml +1 -1
- package/manifests/valkey/valkey-nodeport.yaml +17 -0
- package/package.json +3 -3
- package/scripts/ipxe-setup.sh +52 -49
- package/scripts/k3s-node-setup.sh +84 -68
- package/scripts/lxd-vm-setup.sh +193 -8
- package/scripts/maas-nat-firewalld.sh +145 -0
- package/src/cli/baremetal.js +115 -93
- package/src/cli/cluster.js +548 -221
- package/src/cli/deploy.js +131 -166
- package/src/cli/fs.js +11 -3
- package/src/cli/index.js +75 -17
- package/src/cli/lxd.js +1034 -240
- package/src/cli/monitor.js +9 -3
- package/src/cli/release.js +72 -36
- package/src/cli/repository.js +10 -16
- package/src/cli/run.js +70 -53
- package/src/cli/secrets.js +11 -2
- package/src/client/components/core/Auth.js +4 -3
- package/src/client/components/core/ClientEvents.js +76 -0
- package/src/client/components/core/EventBus.js +4 -0
- package/src/client/components/core/Modal.js +82 -41
- package/src/db/DataBaseProvider.js +9 -9
- package/src/db/mariadb/MariaDB.js +2 -1
- package/src/db/mongo/MongoBootstrap.js +592 -522
- package/src/db/mongo/MongooseDB.js +19 -15
- package/src/index.js +1 -1
- package/src/server/conf.js +62 -15
- package/src/server/proxy.js +9 -2
- package/src/server/start.js +7 -3
- package/src/server/valkey.js +2 -0
- package/bin/file.js +0 -220
- package/bin/vs.js +0 -74
- package/bin/zed.js +0 -84
package/src/cli/cluster.js
CHANGED
|
@@ -24,7 +24,6 @@ const logger = loggerFactory(import.meta);
|
|
|
24
24
|
*/
|
|
25
25
|
class UnderpostCluster {
|
|
26
26
|
static API = {
|
|
27
|
-
|
|
28
27
|
/**
|
|
29
28
|
* @method init
|
|
30
29
|
* @description Initializes and configures the Kubernetes cluster based on provided options.
|
|
@@ -34,7 +33,7 @@ class UnderpostCluster {
|
|
|
34
33
|
* @param {object} [options] - Configuration options for cluster initialization.
|
|
35
34
|
* @param {boolean} [options.mongodb=false] - Deploy MongoDB.
|
|
36
35
|
* @param {boolean} [options.mongodb4=false] - Deploy MongoDB 4.4.
|
|
37
|
-
* @param {
|
|
36
|
+
* @param {string} [options.serviceHost=''] - Set a custom host/IP for exposed MongoDB and Valkey clients.
|
|
38
37
|
* @param {boolean} [options.mariadb=false] - Deploy MariaDB.
|
|
39
38
|
* @param {boolean} [options.mysql=false] - Deploy MySQL.
|
|
40
39
|
* @param {boolean} [options.postgresql=false] - Deploy PostgreSQL.
|
|
@@ -65,6 +64,12 @@ class UnderpostCluster {
|
|
|
65
64
|
* @param {boolean} [options.removeVolumeHostPaths=false] - Remove data from host paths used by Persistent Volumes.
|
|
66
65
|
* @param {string} [options.hosts] - Set custom hosts entries.
|
|
67
66
|
* @param {string} [options.replicas] - Set the number of replicas for certain deployments.
|
|
67
|
+
* @param {boolean} [options.nodePort=false] - Expose enabled ready services (e.g. MongoDB 4.4, Valkey)
|
|
68
|
+
* to the host/public network via their NodePort Service manifest. The node port value lives directly
|
|
69
|
+
* in each manifest (mongodb-4.4/mongodb-nodeport.yaml, valkey/valkey-nodeport.yaml).
|
|
70
|
+
* @param {string} [options.nodeSelector=''] - Pin the just-deployed StatefulSet (MongoDB 4.4 / Valkey)
|
|
71
|
+
* to a specific Kubernetes node by name (e.g. 'localhost.localdomain'). Applied via a
|
|
72
|
+
* `kubernetes.io/hostname` nodeSelector patch once the workload reports Ready.
|
|
68
73
|
* @memberof UnderpostCluster
|
|
69
74
|
*/
|
|
70
75
|
async init(
|
|
@@ -72,7 +77,7 @@ class UnderpostCluster {
|
|
|
72
77
|
options = {
|
|
73
78
|
mongodb: false,
|
|
74
79
|
mongodb4: false,
|
|
75
|
-
|
|
80
|
+
serviceHost: '',
|
|
76
81
|
mariadb: false,
|
|
77
82
|
mysql: false,
|
|
78
83
|
postgresql: false,
|
|
@@ -103,18 +108,22 @@ class UnderpostCluster {
|
|
|
103
108
|
removeVolumeHostPaths: false,
|
|
104
109
|
hosts: '',
|
|
105
110
|
replicas: '',
|
|
111
|
+
nodePort: false,
|
|
112
|
+
nodeSelector: '',
|
|
106
113
|
},
|
|
107
114
|
) {
|
|
108
115
|
if (options.initHost) return Underpost.cluster.initHost();
|
|
109
116
|
|
|
110
117
|
if (options.uninstallHost) return Underpost.cluster.uninstallHost();
|
|
111
118
|
|
|
112
|
-
if (options.config) return Underpost.cluster.config();
|
|
119
|
+
if (options.config) return options.k3s ? Underpost.cluster.configMinimalK3s() : Underpost.cluster.config();
|
|
113
120
|
|
|
114
|
-
if (options.chown) return Underpost.cluster.chown();
|
|
121
|
+
if (options.chown) return Underpost.cluster.chown(options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind');
|
|
115
122
|
|
|
116
123
|
const npmRoot = getNpmRootPath();
|
|
117
124
|
const underpostRoot = options.dev ? '.' : `${npmRoot}/underpost`;
|
|
125
|
+
const serviceHostInput = `${options.serviceHost || ''}`.trim();
|
|
126
|
+
const serviceHost = Underpost.cluster.resolveServiceHost(options);
|
|
118
127
|
|
|
119
128
|
if (options.listPods) return console.table(Underpost.kubectl.get(podName ?? undefined));
|
|
120
129
|
// Set default namespace if not specified
|
|
@@ -141,13 +150,18 @@ class UnderpostCluster {
|
|
|
141
150
|
return;
|
|
142
151
|
}
|
|
143
152
|
|
|
144
|
-
//
|
|
153
|
+
// Route reset to the per-type method. Each only touches what its runtime owns.
|
|
145
154
|
if (options.reset) {
|
|
146
|
-
|
|
147
|
-
|
|
155
|
+
if (options.k3s)
|
|
156
|
+
return await Underpost.cluster.safeResetK3s({ underpostRoot, resetMode: options.resetMode || 'full' });
|
|
157
|
+
if (options.kubeadm)
|
|
158
|
+
return await Underpost.cluster.safeResetKubeadm({
|
|
159
|
+
underpostRoot,
|
|
160
|
+
removeVolumeHostPaths: options.removeVolumeHostPaths,
|
|
161
|
+
});
|
|
162
|
+
return await Underpost.cluster.safeResetKind({
|
|
148
163
|
underpostRoot,
|
|
149
164
|
removeVolumeHostPaths: options.removeVolumeHostPaths,
|
|
150
|
-
clusterType,
|
|
151
165
|
});
|
|
152
166
|
}
|
|
153
167
|
|
|
@@ -161,20 +175,32 @@ class UnderpostCluster {
|
|
|
161
175
|
});
|
|
162
176
|
}
|
|
163
177
|
|
|
164
|
-
// Check if a cluster (Kind, Kubeadm, or K3s) is already initialized
|
|
165
|
-
|
|
166
|
-
const
|
|
167
|
-
|
|
168
|
-
|
|
178
|
+
// Check if a cluster (Kind, Kubeadm, or K3s) is already initialized by
|
|
179
|
+
// inspecting nodes rather than probing add-on pods. See detectClusterRuntime.
|
|
180
|
+
const runtime = Underpost.cluster.detectClusterRuntime();
|
|
181
|
+
const alreadyCluster = runtime.type !== null;
|
|
182
|
+
if (alreadyCluster) {
|
|
183
|
+
logger.info(
|
|
184
|
+
`Detected existing ${runtime.type} cluster (${runtime.ready ? 'Ready' : 'NotReady'}); skipping initialization.`,
|
|
185
|
+
);
|
|
186
|
+
}
|
|
169
187
|
|
|
170
188
|
// --- Kubeadm/Kind/K3s Cluster Initialization ---
|
|
171
|
-
|
|
172
|
-
|
|
189
|
+
// Host config is applied per cluster type (not shared) so the K3s path can
|
|
190
|
+
// stay minimal and avoid the Docker/containerd/kubelet setup it does not need.
|
|
191
|
+
if (!alreadyCluster) {
|
|
173
192
|
if (options.k3s) {
|
|
193
|
+
// K3s is self-contained (bundles containerd, kubelet, CNI, CoreDNS).
|
|
194
|
+
// Apply ONLY minimal host config — see configMinimalK3s.
|
|
195
|
+
Underpost.cluster.configMinimalK3s();
|
|
174
196
|
logger.info('Initializing K3s control plane...');
|
|
175
197
|
// Install K3s
|
|
176
198
|
logger.info('Installing K3s...');
|
|
177
|
-
|
|
199
|
+
// Disable the bundled traefik ingress and servicelb (Klipper) load
|
|
200
|
+
// balancer. The platform exposes services explicitly via Project
|
|
201
|
+
// Contour / Envoy and NodePort services (see --node-port); leaving the
|
|
202
|
+
// K3s built-ins enabled would bind the same host ports and conflict.
|
|
203
|
+
shellExec(`curl -sfL https://get.k3s.io | sh -s - --disable=traefik --disable=servicelb`);
|
|
178
204
|
logger.info('K3s installation completed.');
|
|
179
205
|
|
|
180
206
|
Underpost.cluster.chown('k3s');
|
|
@@ -182,7 +208,11 @@ class UnderpostCluster {
|
|
|
182
208
|
logger.info('Waiting for K3s to be ready...');
|
|
183
209
|
shellExec(`sudo systemctl is-active --wait k3s || sudo systemctl wait --for=active k3s.service`);
|
|
184
210
|
logger.info('K3s service is active.');
|
|
211
|
+
// K3s manages its own CNI (flannel) and iptables rules. nat-iptables.sh
|
|
212
|
+
// is neither needed nor desirable for a single K3s node in a VM.
|
|
185
213
|
} else if (options.kubeadm) {
|
|
214
|
+
Underpost.cluster.config();
|
|
215
|
+
Underpost.cluster.natSetup({ underpostRoot });
|
|
186
216
|
logger.info('Initializing Kubeadm control plane...');
|
|
187
217
|
// Set default values if not provided
|
|
188
218
|
const podNetworkCidr = options.podNetworkCidr || '192.168.0.0/16';
|
|
@@ -218,12 +248,14 @@ class UnderpostCluster {
|
|
|
218
248
|
// Untaint control plane node to allow scheduling pods
|
|
219
249
|
const nodeName = os.hostname();
|
|
220
250
|
shellExec(`kubectl taint nodes ${nodeName} node-role.kubernetes.io/control-plane:NoSchedule-`);
|
|
221
|
-
// Install local-path-provisioner for dynamic PVCs
|
|
251
|
+
// Install local-path-provisioner for dynamic PVCs
|
|
222
252
|
logger.info('Installing local-path-provisioner...');
|
|
223
253
|
shellExec(
|
|
224
254
|
`kubectl apply -f https://cdn.jsdelivr.net/gh/rancher/local-path-provisioner@master/deploy/local-path-storage.yaml`,
|
|
225
255
|
);
|
|
226
256
|
} else {
|
|
257
|
+
Underpost.cluster.config();
|
|
258
|
+
Underpost.cluster.natSetup({ underpostRoot });
|
|
227
259
|
// Kind cluster initialization (default for development)
|
|
228
260
|
logger.info('Initializing Kind cluster...');
|
|
229
261
|
const devReplicaCount = Math.max(Number(options.replicas) || MONGODB_DEFAULT_REPLICA_COUNT, 3);
|
|
@@ -237,14 +269,18 @@ class UnderpostCluster {
|
|
|
237
269
|
} catch (error) {
|
|
238
270
|
const kindCreateErrText = `${error?.message || ''}\n${error?.stderr || ''}`;
|
|
239
271
|
if (kindCreateErrText.includes('all predefined address pools have been fully subnetted')) {
|
|
240
|
-
logger.warn(
|
|
272
|
+
logger.warn(
|
|
273
|
+
'Docker address pool exhausted while creating Kind cluster. Running cleanup and retrying once...',
|
|
274
|
+
);
|
|
241
275
|
Underpost.cluster.recoverKindDockerNetworks();
|
|
242
276
|
try {
|
|
243
277
|
shellExec(kindCreateCmd);
|
|
244
278
|
} catch (retryError) {
|
|
245
279
|
const retryErrText = `${retryError?.message || ''}\n${retryError?.stderr || ''}`;
|
|
246
280
|
if (retryErrText.includes('all predefined address pools have been fully subnetted')) {
|
|
247
|
-
logger.warn(
|
|
281
|
+
logger.warn(
|
|
282
|
+
'Kind retry still failed from pool exhaustion. Applying Docker daemon address-pool config and retrying once more...',
|
|
283
|
+
);
|
|
248
284
|
Underpost.cluster.ensureDockerDefaultAddressPools();
|
|
249
285
|
shellExec(kindCreateCmd);
|
|
250
286
|
} else {
|
|
@@ -257,7 +293,6 @@ class UnderpostCluster {
|
|
|
257
293
|
}
|
|
258
294
|
Underpost.cluster.chown('kind'); // Pass 'kind' to chown
|
|
259
295
|
}
|
|
260
|
-
Underpost.cluster.natSetup({ underpostRoot });
|
|
261
296
|
}
|
|
262
297
|
|
|
263
298
|
// --- Optional Component Deployments (Databases, Ingress, Cert-Manager) ---
|
|
@@ -304,7 +339,23 @@ EOF
|
|
|
304
339
|
if (options.pullImage) Underpost.cluster.pullImage('valkey/valkey:latest', options);
|
|
305
340
|
shellExec(`kubectl delete statefulset valkey-service -n ${options.namespace} --ignore-not-found`);
|
|
306
341
|
shellExec(`kubectl apply -k ${underpostRoot}/manifests/valkey -n ${options.namespace}`);
|
|
307
|
-
await Underpost.test.statusMonitor('valkey-service', 'Running', 'pods', 1000, 60 * 10);
|
|
342
|
+
const valkeyReady = await Underpost.test.statusMonitor('valkey-service', 'Running', 'pods', 1000, 60 * 10);
|
|
343
|
+
// Expose valkey to the host/public network only once the pod is ready.
|
|
344
|
+
// The node port (32079) is set directly in the manifest.
|
|
345
|
+
if (valkeyReady && options.nodePort)
|
|
346
|
+
shellExec(`kubectl apply -f ${underpostRoot}/manifests/valkey/valkey-nodeport.yaml -n ${options.namespace}`);
|
|
347
|
+
if (valkeyReady && options.nodeSelector)
|
|
348
|
+
Underpost.cluster.pinToNode({
|
|
349
|
+
name: 'valkey-service',
|
|
350
|
+
namespace: options.namespace,
|
|
351
|
+
node: options.nodeSelector,
|
|
352
|
+
});
|
|
353
|
+
if (valkeyReady && serviceHost)
|
|
354
|
+
Underpost.cluster.syncServiceConnectionEnv({
|
|
355
|
+
serviceHost,
|
|
356
|
+
valkey: true,
|
|
357
|
+
options,
|
|
358
|
+
});
|
|
308
359
|
}
|
|
309
360
|
if (options.ipfs) {
|
|
310
361
|
await Underpost.ipfs.deploy(options, underpostRoot);
|
|
@@ -337,37 +388,75 @@ EOF
|
|
|
337
388
|
}
|
|
338
389
|
if (options.mongodb4) {
|
|
339
390
|
if (options.pullImage) Underpost.cluster.pullImage('mongo:4.4', options);
|
|
391
|
+
shellExec(`kubectl delete statefulset mongodb -n ${options.namespace} --ignore-not-found`);
|
|
340
392
|
shellExec(`kubectl apply -k ${underpostRoot}/manifests/mongodb-4.4 -n ${options.namespace}`);
|
|
341
393
|
|
|
342
|
-
const
|
|
394
|
+
const statefulSetName = 'mongodb';
|
|
395
|
+
const podName = 'mongodb-0';
|
|
343
396
|
|
|
344
|
-
const successInstance = await Underpost.test.statusMonitor(
|
|
397
|
+
const successInstance = await Underpost.test.statusMonitor(podName);
|
|
345
398
|
|
|
346
399
|
if (successInstance) {
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
400
|
+
// mongod only accepts a member host it can recognize as itself (the isSelf check):
|
|
401
|
+
// it must match a local interface IP or be reachable back at that address. A pod-external
|
|
402
|
+
// LAN IP / NodePort address is neither bound in the pod netns nor routable back from it,
|
|
403
|
+
// so reconfiguring to it fails with NodeNotFound ("...maps to this node") even under
|
|
404
|
+
// force. Bootstrap on localhost; only advertise a non-localhost host when the node can
|
|
405
|
+
// self-verify it, and tolerate the failure otherwise so bootstrap stays idempotent.
|
|
406
|
+
// Clients reaching the set through an external IP/NodePort must use directConnection=true,
|
|
407
|
+
// which ignores the advertised member host (see MongooseDB.buildUri).
|
|
408
|
+
const rsHost = serviceHostInput || (options.dev ? '127.0.0.1' : 'mongodb-0.mongodb-service');
|
|
409
|
+
const memberHost = `${rsHost}:27017`;
|
|
410
|
+
const initEval = [
|
|
411
|
+
`try { rs.initiate({ _id: "rs0", members: [{ _id: 0, host: "localhost:27017" }] }); }`,
|
|
412
|
+
`catch (e) { if (!String(e).includes("already initialized") && !String(e).includes("AlreadyInitialized")) throw e; }`,
|
|
413
|
+
`for (var i = 0; i < 30; i++) { if (db.isMaster().ismaster) break; sleep(1000); }`,
|
|
414
|
+
memberHost === 'localhost:27017'
|
|
415
|
+
? ``
|
|
416
|
+
: `try { var c = rs.conf(); c.members[0].host = "${memberHost}"; rs.reconfig(c, { force: true }); print("RS_HOST_SET ${memberHost}"); } catch (e) { if (String(e).includes("NodeNotFound") || String(e).includes("maps to this node")) { print("RS_HOST_SKIPPED ${memberHost} (not self-reachable from pod; clients must use directConnection=true)"); } else { throw e; } }`,
|
|
417
|
+
]
|
|
418
|
+
.filter(Boolean)
|
|
419
|
+
.join(' ');
|
|
420
|
+
|
|
421
|
+
shellExec(`sudo kubectl exec -i ${podName} -n ${options.namespace} -- mongo --quiet --eval '${initEval}'`);
|
|
422
|
+
|
|
423
|
+
// Only expose mongos to the host/public network once the instance is
|
|
424
|
+
// confirmed ready and the replica set is initiated. The node port (32017)
|
|
425
|
+
// is set directly in the manifest.
|
|
426
|
+
if (options.nodePort)
|
|
427
|
+
shellExec(
|
|
428
|
+
`kubectl apply -f ${underpostRoot}/manifests/mongodb-4.4/mongodb-nodeport.yaml -n ${options.namespace}`,
|
|
429
|
+
);
|
|
430
|
+
if (options.nodeSelector)
|
|
431
|
+
Underpost.cluster.pinToNode({
|
|
432
|
+
name: statefulSetName,
|
|
433
|
+
namespace: options.namespace,
|
|
434
|
+
node: options.nodeSelector,
|
|
435
|
+
});
|
|
436
|
+
if (serviceHost)
|
|
437
|
+
Underpost.cluster.syncServiceConnectionEnv({
|
|
438
|
+
serviceHost,
|
|
439
|
+
mongodb: true,
|
|
440
|
+
options,
|
|
441
|
+
});
|
|
359
442
|
}
|
|
360
443
|
} else if (options.mongodb) {
|
|
361
444
|
const clusterType = options.k3s ? 'k3s' : options.kubeadm ? 'kubeadm' : 'kind';
|
|
362
445
|
await MongoBootstrap.initReplicaSet({
|
|
363
446
|
namespace: options.namespace,
|
|
364
447
|
replicaCount: Number(options.replicas) || MONGODB_DEFAULT_REPLICA_COUNT,
|
|
365
|
-
|
|
448
|
+
hostList: serviceHostInput,
|
|
366
449
|
pullImage: options.pullImage,
|
|
367
450
|
reset: options.reset,
|
|
368
451
|
clusterType,
|
|
369
452
|
underpostRoot,
|
|
370
453
|
});
|
|
454
|
+
if (serviceHost)
|
|
455
|
+
Underpost.cluster.syncServiceConnectionEnv({
|
|
456
|
+
serviceHost,
|
|
457
|
+
mongodb: true,
|
|
458
|
+
options,
|
|
459
|
+
});
|
|
371
460
|
}
|
|
372
461
|
|
|
373
462
|
if (options.contour) {
|
|
@@ -402,6 +491,132 @@ EOF
|
|
|
402
491
|
}
|
|
403
492
|
},
|
|
404
493
|
|
|
494
|
+
/**
|
|
495
|
+
* @method detectClusterRuntime
|
|
496
|
+
* @description Detects an already-initialized cluster by inspecting Kubernetes
|
|
497
|
+
* nodes, and classifies its runtime. Nodes are authoritative and stable, unlike
|
|
498
|
+
* add-on pods: the previous check keyed off pod names (`calico-kube-controllers`,
|
|
499
|
+
* `kube-apiserver-kind-control-plane`, `svclb-traefik`), whose presence and
|
|
500
|
+
* readiness are timing- and config-dependent. Disabling servicelb removes the
|
|
501
|
+
* `svclb-traefik` pods entirely, and CNI/controller pods report NotReady for a
|
|
502
|
+
* window right after init — both of which made re-runs misdetect the cluster
|
|
503
|
+
* state. Classification relies on stable node attributes:
|
|
504
|
+
* - k3s: the node VERSION carries a `+k3s` build suffix (e.g. v1.30.5+k3s1).
|
|
505
|
+
* - kind: kind names every node `<cluster>-control-plane` / `<cluster>-worker`.
|
|
506
|
+
* - kubeadm: a real control-plane node that is neither of the above.
|
|
507
|
+
* @returns {{ type: ('k3s'|'kubeadm'|'kind'|null), ready: boolean, nodes: Array<object> }}
|
|
508
|
+
* `type` is the detected runtime (null when no cluster exists); `ready` is true
|
|
509
|
+
* when at least one node reports STATUS=Ready.
|
|
510
|
+
* @memberof UnderpostCluster
|
|
511
|
+
*/
|
|
512
|
+
detectClusterRuntime() {
|
|
513
|
+
const nodes = Underpost.kubectl.get('', 'nodes');
|
|
514
|
+
if (!nodes.length) return { type: null, ready: false, nodes: [] };
|
|
515
|
+
|
|
516
|
+
// STATUS can be a comma-joined list (e.g. "Ready,SchedulingDisabled").
|
|
517
|
+
const ready = nodes.some((n) => `${n.STATUS || ''}`.split(',').includes('Ready'));
|
|
518
|
+
|
|
519
|
+
let type;
|
|
520
|
+
if (nodes.some((n) => `${n.VERSION || ''}`.includes('+k3s'))) type = 'k3s';
|
|
521
|
+
else if (nodes.some((n) => `${n.NAME || ''}`.includes('-control-plane') || `${n.NAME || ''}`.includes('kind')))
|
|
522
|
+
type = 'kind';
|
|
523
|
+
else type = 'kubeadm';
|
|
524
|
+
|
|
525
|
+
return { type, ready, nodes };
|
|
526
|
+
},
|
|
527
|
+
|
|
528
|
+
/**
|
|
529
|
+
* @method pinToNode
|
|
530
|
+
* @description Pins a workload to a specific Kubernetes node by patching its
|
|
531
|
+
* pod template with a `kubernetes.io/hostname` nodeSelector. General-purpose;
|
|
532
|
+
* currently used to place the MongoDB 4.4 / Valkey StatefulSets on a chosen
|
|
533
|
+
* node (`--node-selector`). The patch triggers a rolling reschedule onto the
|
|
534
|
+
* target node.
|
|
535
|
+
* @param {object} params
|
|
536
|
+
* @param {string} [params.kind='statefulset'] - Workload kind to patch.
|
|
537
|
+
* @param {string} params.name - Workload name.
|
|
538
|
+
* @param {string} params.namespace - Target namespace.
|
|
539
|
+
* @param {string} params.node - Target node name (matches `kubernetes.io/hostname`).
|
|
540
|
+
* @memberof UnderpostCluster
|
|
541
|
+
*/
|
|
542
|
+
pinToNode({ kind = 'statefulset', name, namespace, node }) {
|
|
543
|
+
logger.info(`Pinning ${kind}/${name} to node '${node}' (namespace: ${namespace}).`);
|
|
544
|
+
const patch = JSON.stringify({
|
|
545
|
+
spec: { template: { spec: { nodeSelector: { 'kubernetes.io/hostname': node } } } },
|
|
546
|
+
});
|
|
547
|
+
shellExec(`kubectl patch ${kind} ${name} -n ${namespace} --type merge -p '${patch}'`);
|
|
548
|
+
},
|
|
549
|
+
|
|
550
|
+
/**
|
|
551
|
+
* @method resolveServiceHost
|
|
552
|
+
* @description Resolves a shared single-host override used by exposed service clients.
|
|
553
|
+
* @param {object} [options={}] - Cluster options.
|
|
554
|
+
* @returns {string} A single host override, or an empty string when unset / not reusable.
|
|
555
|
+
* @memberof UnderpostCluster
|
|
556
|
+
*/
|
|
557
|
+
resolveServiceHost(options = {}) {
|
|
558
|
+
const candidate = `${options.serviceHost || ''}`.trim();
|
|
559
|
+
return candidate && !candidate.includes(',') ? candidate : '';
|
|
560
|
+
},
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* @method upsertEnvVar
|
|
564
|
+
* @description Replaces or appends one env var assignment in raw env file text.
|
|
565
|
+
* @param {string} envText - Existing env file contents.
|
|
566
|
+
* @param {string} key - Env var name.
|
|
567
|
+
* @param {string} value - Env var value.
|
|
568
|
+
* @returns {string} Updated env file contents.
|
|
569
|
+
* @memberof UnderpostCluster
|
|
570
|
+
*/
|
|
571
|
+
upsertEnvVar(envText, key, value) {
|
|
572
|
+
const nextEntry = `${key}=${value}`;
|
|
573
|
+
const envKeyPattern = new RegExp(`^${key}=.*$`, 'm');
|
|
574
|
+
if (envKeyPattern.test(envText)) return envText.replace(envKeyPattern, nextEntry);
|
|
575
|
+
|
|
576
|
+
const trimmedEnvText = envText.replace(/\s*$/, '');
|
|
577
|
+
return `${trimmedEnvText}${trimmedEnvText ? '\n' : ''}${nextEntry}\n`;
|
|
578
|
+
},
|
|
579
|
+
|
|
580
|
+
/**
|
|
581
|
+
* @method syncServiceConnectionEnv
|
|
582
|
+
* @description Persists exposed service connection hosts to the active deploy env files.
|
|
583
|
+
* Currently applies only to MongoDB (`DB_HOST`) and Valkey (`VALKEY_HOST`).
|
|
584
|
+
* @param {object} params
|
|
585
|
+
* @param {string} params.serviceHost - Shared exposed host/IP.
|
|
586
|
+
* @param {boolean} [params.mongodb=false] - Update MongoDB runtime host.
|
|
587
|
+
* @param {boolean} [params.valkey=false] - Update Valkey runtime host.
|
|
588
|
+
* @param {object} [params.options={}] - Cluster options used to infer the active env.
|
|
589
|
+
* @memberof UnderpostCluster
|
|
590
|
+
*/
|
|
591
|
+
syncServiceConnectionEnv({ serviceHost, mongodb = false, valkey = false, options = {} }) {
|
|
592
|
+
if (!serviceHost) return;
|
|
593
|
+
|
|
594
|
+
const updates = {};
|
|
595
|
+
if (mongodb) updates.DB_HOST = `mongodb://${serviceHost}:27017`;
|
|
596
|
+
if (valkey) updates.VALKEY_HOST = serviceHost;
|
|
597
|
+
if (Object.keys(updates).length === 0) return;
|
|
598
|
+
|
|
599
|
+
const deployId = process.env.DEPLOY_ID || process.env.DEFAULT_DEPLOY_ID || 'dd-default';
|
|
600
|
+
const envName = process.env.NODE_ENV || (options.dev ? 'development' : 'production');
|
|
601
|
+
const envPaths = [`./engine-private/conf/${deployId}/.env.${envName}`, `./.env.${envName}`, `./.env`].filter(
|
|
602
|
+
(envPath, index, paths) => fs.existsSync(envPath) && paths.indexOf(envPath) === index,
|
|
603
|
+
);
|
|
604
|
+
|
|
605
|
+
if (envPaths.length === 0) {
|
|
606
|
+
logger.warn(`No env files found to persist service host override for deploy '${deployId}' (${envName}).`);
|
|
607
|
+
return;
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
for (const envPath of envPaths) {
|
|
611
|
+
let envText = fs.readFileSync(envPath, 'utf8');
|
|
612
|
+
for (const [key, value] of Object.entries(updates))
|
|
613
|
+
envText = Underpost.cluster.upsertEnvVar(envText, key, value);
|
|
614
|
+
fs.writeFileSync(envPath, envText, 'utf8');
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
logger.info(`Persisted service host override for ${Object.keys(updates).join(', ')} to ${envPaths.join(', ')}`);
|
|
618
|
+
},
|
|
619
|
+
|
|
405
620
|
/**
|
|
406
621
|
* @method pullImage
|
|
407
622
|
* @description Pulls a container image using the appropriate runtime based on the cluster type.
|
|
@@ -422,7 +637,13 @@ EOF
|
|
|
422
637
|
`for node in $(kind get nodes); do cat ${tarPath} | docker exec -i $node ctr --namespace=k8s.io images import -; done`,
|
|
423
638
|
);
|
|
424
639
|
shellExec(`rm -f ${tarPath}`);
|
|
425
|
-
} else if (options.
|
|
640
|
+
} else if (options.k3s) {
|
|
641
|
+
// K3s uses its own embedded containerd socket, not the host-level one
|
|
642
|
+
// used by kubeadm/containerd installations.
|
|
643
|
+
shellExec(
|
|
644
|
+
`sudo env PATH="$PATH:/usr/local/bin:/usr/bin" crictl --runtime-endpoint unix:///run/k3s/containerd/containerd.sock pull ${image}`,
|
|
645
|
+
);
|
|
646
|
+
} else if (options.kubeadm) {
|
|
426
647
|
// Kubeadm / K3s: use crictl to pull directly into the active CRI runtime.
|
|
427
648
|
// crictl is not in sudo's secure_path; pass full PATH through env.
|
|
428
649
|
// Point crictl at CRI-O when the socket exists, otherwise fall back to containerd.
|
|
@@ -451,8 +672,12 @@ EOF
|
|
|
451
672
|
const { underpostRoot } = options;
|
|
452
673
|
console.log('Applying host configuration: SELinux, Docker, Containerd, and Sysctl settings.');
|
|
453
674
|
// Disable SELinux (permissive mode)
|
|
454
|
-
shellExec(`sudo setenforce 0
|
|
455
|
-
|
|
675
|
+
shellExec(`sudo setenforce 0`, {
|
|
676
|
+
silentOnError: true,
|
|
677
|
+
});
|
|
678
|
+
shellExec(`sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config`, {
|
|
679
|
+
silentOnError: true,
|
|
680
|
+
});
|
|
456
681
|
|
|
457
682
|
// Enable and start Docker and Kubelet services
|
|
458
683
|
shellExec(`sudo systemctl enable --now docker`); // Docker might not be needed for K3s
|
|
@@ -480,7 +705,66 @@ EOF
|
|
|
480
705
|
shellExec(`sudo sysctl -w fs.inotify.max_user_watches=2099999999`);
|
|
481
706
|
shellExec(`sudo sysctl -w fs.inotify.max_user_instances=2099999999`);
|
|
482
707
|
shellExec(`sudo sysctl -w fs.inotify.max_queued_events=2099999999`);
|
|
708
|
+
},
|
|
483
709
|
|
|
710
|
+
/**
|
|
711
|
+
* @method configMinimalK3s
|
|
712
|
+
* @description Minimal host configuration for a K3s node. K3s is fully
|
|
713
|
+
* self-contained — it ships its own containerd, kubelet, CNI (flannel),
|
|
714
|
+
* CoreDNS and traefik. It therefore needs NONE of the Docker /
|
|
715
|
+
* standalone-containerd / standalone-kubelet setup that `config()` applies
|
|
716
|
+
* for kind and kubeadm. In a fresh LXD VM those packages do not exist, so
|
|
717
|
+
* `config()` there is both redundant and a source of errors.
|
|
718
|
+
*
|
|
719
|
+
* This applies only what K3s genuinely requires, and every step is guarded
|
|
720
|
+
* so it is a no-op when the relevant tooling is absent (e.g. minimal images
|
|
721
|
+
* without SELinux userspace):
|
|
722
|
+
* - SELinux → permissive (only if SELinux tooling is present).
|
|
723
|
+
* - swap off (Kubernetes best practice).
|
|
724
|
+
* - br_netfilter + bridge/forward sysctls (pod networking).
|
|
725
|
+
* - inotify limits.
|
|
726
|
+
* @memberof UnderpostCluster
|
|
727
|
+
*/
|
|
728
|
+
configMinimalK3s() {
|
|
729
|
+
console.log('Applying minimal K3s host configuration (firewalld, SELinux, swap, sysctl).');
|
|
730
|
+
|
|
731
|
+
// Disable firewalld. K3s manages its own iptables rules; firewalld
|
|
732
|
+
// closes 6443/tcp (supervisor + API) and 8472/udp (flannel VXLAN) by
|
|
733
|
+
// default on RHEL/Rocky, which makes k3s-agent hang on `systemctl start`
|
|
734
|
+
// forever (the upstream unit ships TimeoutStartSec=0).
|
|
735
|
+
shellExec(`if systemctl is-active --quiet firewalld; then sudo systemctl disable --now firewalld; fi`);
|
|
736
|
+
|
|
737
|
+
// SELinux → permissive, but only when the tooling exists. Rocky has it;
|
|
738
|
+
// minimal LXD images may not. K3s also installs k3s-selinux for enforcing
|
|
739
|
+
// mode, so this is a best-effort dev convenience, not a hard requirement.
|
|
740
|
+
shellExec(`if command -v setenforce >/dev/null 2>&1; then sudo setenforce 0; fi`, {
|
|
741
|
+
silentOnError: true,
|
|
742
|
+
});
|
|
743
|
+
shellExec(
|
|
744
|
+
`if [ -f /etc/selinux/config ]; then sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config; fi`,
|
|
745
|
+
{ silentOnError: true },
|
|
746
|
+
);
|
|
747
|
+
|
|
748
|
+
// Disable swap. `swapoff -a` is a no-op without swap; the sed only edits
|
|
749
|
+
// fstab when a swap line is present.
|
|
750
|
+
shellExec(`sudo swapoff -a`);
|
|
751
|
+
shellExec(`sudo sed -i '/swap/d' /etc/fstab`);
|
|
752
|
+
|
|
753
|
+
// Pod networking: ensure br_netfilter is loaded and the bridge/forward
|
|
754
|
+
// sysctls are set. K3s + flannel depend on these.
|
|
755
|
+
shellExec(
|
|
756
|
+
`if command -v lsmod >/dev/null 2>&1 && command -v modprobe >/dev/null 2>&1; then if ! lsmod | grep -q '^br_netfilter'; then sudo modprobe br_netfilter || true; fi; fi`,
|
|
757
|
+
);
|
|
758
|
+
shellExec(
|
|
759
|
+
`echo 'net.bridge.bridge-nf-call-iptables = 1
|
|
760
|
+
net.bridge.bridge-nf-call-ip6tables = 1
|
|
761
|
+
net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-k3s.conf > /dev/null`,
|
|
762
|
+
);
|
|
763
|
+
shellExec(`sudo sysctl --system`);
|
|
764
|
+
|
|
765
|
+
// inotify limits — many pods/watchers. Conservative, sane values.
|
|
766
|
+
shellExec(`sudo sysctl -w fs.inotify.max_user_instances=1024`);
|
|
767
|
+
shellExec(`sudo sysctl -w fs.inotify.max_user_watches=1048576`);
|
|
484
768
|
},
|
|
485
769
|
|
|
486
770
|
/**
|
|
@@ -490,33 +774,16 @@ EOF
|
|
|
490
774
|
* required for Kubernetes cluster communication. It is designed to work with kubeadm and k3s clusters, ensuring that
|
|
491
775
|
* traffic through Linux bridges is processed by iptables, which is crucial for CNI plugins to function correctly.
|
|
492
776
|
* The method also applies NAT iptables rules and configures firewalld for Kubernetes, which is required for multi-machine kubeadm inter-node communication.
|
|
493
|
-
* Note: This method
|
|
777
|
+
* Note: This method must be called before kubeadm init / kind create so that br_netfilter is loaded and kernel networking is ready when the control plane starts.
|
|
494
778
|
* @param {object} [options] - Configuration options for NAT setup.
|
|
495
779
|
* @param {string} [options.underpostRoot] - The root path of the underpost project, used to locate the nat-iptables.sh script.
|
|
496
780
|
* @memberof UnderpostCluster
|
|
497
781
|
*/
|
|
498
782
|
natSetup(options = { underpostRoot: '.' }) {
|
|
499
783
|
const { underpostRoot } = options;
|
|
500
|
-
//
|
|
501
|
-
//
|
|
502
|
-
|
|
503
|
-
`/etc/sysctl.d/k8s.conf`,
|
|
504
|
-
`/etc/sysctl.d/99-k8s-ipforward.conf`,
|
|
505
|
-
`/etc/sysctl.d/99-k8s.conf`,
|
|
506
|
-
])
|
|
507
|
-
shellExec(
|
|
508
|
-
`echo 'net.bridge.bridge-nf-call-iptables = 1
|
|
509
|
-
net.bridge.bridge-nf-call-ip6tables = 1
|
|
510
|
-
net.bridge.bridge-nf-call-arptables = 1
|
|
511
|
-
net.ipv4.ip_forward = 1' | sudo tee ${iptableConfPath}`,
|
|
512
|
-
{ silent: true },
|
|
513
|
-
);
|
|
514
|
-
// shellExec(`sudo sysctl --system`); // Apply sysctl changes immediately
|
|
515
|
-
// Apply NAT iptables rules and configure firewalld for Kubernetes.
|
|
516
|
-
// nat-iptables.sh enables firewalld and opens all required ports; do NOT stop it
|
|
517
|
-
// afterwards — keeping firewalld running with these rules is required for
|
|
518
|
-
// multi-machine kubeadm inter-node communication.
|
|
519
|
-
shellExec(`${underpostRoot}/scripts/nat-iptables.sh`, { silent: true });
|
|
784
|
+
// Loads br_netfilter, applies bridge/forward sysctls, opens firewall ports, enables masquerade.
|
|
785
|
+
// Must run before kubeadm init / kind create so kernel networking is ready.
|
|
786
|
+
shellExec(`${underpostRoot}/scripts/nat-iptables.sh`);
|
|
520
787
|
},
|
|
521
788
|
|
|
522
789
|
/**
|
|
@@ -553,169 +820,231 @@ net.ipv4.ip_forward = 1' | sudo tee ${iptableConfPath}`,
|
|
|
553
820
|
console.log('kubectl config set up successfully.');
|
|
554
821
|
},
|
|
555
822
|
|
|
823
|
+
// Shared reset helpers (internal — used by safeResetKind / safeResetKubeadm).
|
|
824
|
+
|
|
556
825
|
/**
|
|
557
|
-
* @method
|
|
558
|
-
* @description
|
|
559
|
-
*
|
|
560
|
-
* in coredns) by restoring SELinux security contexts and safely cleaning up cluster artifacts.
|
|
561
|
-
* Only the uninstall/delete commands specific to the given clusterType are executed; all other
|
|
562
|
-
* cleanup steps (log truncation, filesystem, network) are always run as generic k8s resets.
|
|
563
|
-
* @param {object} [options] - Configuration options for the reset.
|
|
564
|
-
* @param {string} [options.underpostRoot] - The root path of the underpost project.
|
|
565
|
-
* @param {boolean} [options.removeVolumeHostPaths=false] - Whether to remove data from host paths used by Persistent Volumes.
|
|
566
|
-
* @param {string} [options.clusterType='kind'] - The type of cluster to reset: 'kind', 'kubeadm', or 'k3s'.
|
|
567
|
-
* @memberof UnderpostCluster
|
|
826
|
+
* @method _truncateLargeLogs
|
|
827
|
+
* @description Removes files >1 GiB under /var/log. Best-effort.
|
|
828
|
+
* @private
|
|
568
829
|
*/
|
|
569
|
-
|
|
570
|
-
logger.info('Starting a safe and comprehensive reset of Kubernetes and container environments...');
|
|
571
|
-
|
|
830
|
+
_truncateLargeLogs() {
|
|
572
831
|
try {
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
);
|
|
583
|
-
for (const pathLog of largeLogsFiles
|
|
584
|
-
.split(`\n`)
|
|
585
|
-
.map((p) => p.split(cleanPath)[1])
|
|
586
|
-
.filter((p) => p)) {
|
|
587
|
-
shellExec(`sudo rm -rf ${cleanPath}${pathLog}`);
|
|
588
|
-
}
|
|
589
|
-
} catch (err) {
|
|
590
|
-
logger.warn(` -> Error truncating log files: ${err.message}. Continuing with reset.`);
|
|
832
|
+
const cleanPath = `/var/log/`;
|
|
833
|
+
const largeLogsFiles = shellExec(
|
|
834
|
+
`sudo du -sh ${cleanPath}* | awk '{if ($1 ~ /G$/ && ($1+0) > 1) print}' | sort -rh`,
|
|
835
|
+
{ stdout: true },
|
|
836
|
+
);
|
|
837
|
+
for (const pathLog of largeLogsFiles
|
|
838
|
+
.split('\n')
|
|
839
|
+
.map((p) => p.split(cleanPath)[1])
|
|
840
|
+
.filter((p) => p)) {
|
|
841
|
+
shellExec(`sudo rm -rf ${cleanPath}${pathLog}`);
|
|
591
842
|
}
|
|
843
|
+
} catch (err) {
|
|
844
|
+
logger.warn(` -> Skipped log truncation: ${err.message}`);
|
|
845
|
+
}
|
|
846
|
+
},
|
|
592
847
|
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
848
|
+
/**
|
|
849
|
+
* @method _cleanHostPathPvs
|
|
850
|
+
* @description Wipes contents of every hostPath PV. Destroys live data —
|
|
851
|
+
* only call when `--remove-volume-host-paths` is set.
|
|
852
|
+
* @private
|
|
853
|
+
*/
|
|
854
|
+
_cleanHostPathPvs() {
|
|
855
|
+
try {
|
|
856
|
+
const pvListJson = shellExec(`kubectl get pv -o json || echo '{"items":[]}'`, {
|
|
857
|
+
stdout: true,
|
|
858
|
+
silent: true,
|
|
859
|
+
});
|
|
860
|
+
const pvList = JSON.parse(pvListJson);
|
|
861
|
+
if (!pvList.items || pvList.items.length === 0) {
|
|
862
|
+
logger.info(' -> No PersistentVolumes with hostPath found.');
|
|
863
|
+
return;
|
|
599
864
|
}
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
const
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
});
|
|
606
|
-
const pvList = JSON.parse(pvListJson);
|
|
607
|
-
|
|
608
|
-
if (pvList.items && pvList.items.length > 0) {
|
|
609
|
-
for (const pv of pvList.items) {
|
|
610
|
-
// Check if the PV uses hostPath and delete its contents
|
|
611
|
-
if (pv.spec.hostPath && pv.spec.hostPath.path) {
|
|
612
|
-
const hostPath = pv.spec.hostPath.path;
|
|
613
|
-
logger.info(`Removing data from host path for PV '${pv.metadata.name}': ${hostPath}`);
|
|
614
|
-
shellExec(`sudo rm -rf ${hostPath}/*`);
|
|
615
|
-
}
|
|
616
|
-
}
|
|
617
|
-
} else {
|
|
618
|
-
logger.info('No Persistent Volumes found with hostPath to clean up.');
|
|
619
|
-
}
|
|
620
|
-
} catch (error) {
|
|
621
|
-
logger.error('Failed to clean up Persistent Volumes:', error);
|
|
865
|
+
for (const pv of pvList.items) {
|
|
866
|
+
if (pv.spec.hostPath && pv.spec.hostPath.path) {
|
|
867
|
+
const hostPath = pv.spec.hostPath.path;
|
|
868
|
+
logger.info(` -> Removing PV '${pv.metadata.name}' hostPath: ${hostPath}`);
|
|
869
|
+
shellExec(`sudo rm -rf ${hostPath}/*`);
|
|
622
870
|
}
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
logger.info(' -> Ensuring SELinux is in permissive mode...');
|
|
629
|
-
shellExec(`sudo setenforce 0`);
|
|
630
|
-
shellExec(`sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config`);
|
|
631
|
-
logger.info(' -> Restoring SELinux contexts for container data directories...');
|
|
632
|
-
// The 'restorecon' command corrects file system security contexts.
|
|
633
|
-
shellExec(`sudo restorecon -Rv /var/lib/containerd`);
|
|
634
|
-
shellExec(`sudo restorecon -Rv /var/lib/kubelet`);
|
|
635
|
-
|
|
636
|
-
logger.info(' -> Stopping kubelet, docker, and podman services...');
|
|
637
|
-
shellExec('sudo systemctl stop kubelet');
|
|
638
|
-
shellExec('sudo systemctl stop docker');
|
|
639
|
-
shellExec('sudo systemctl stop podman');
|
|
640
|
-
// Lazy-unmount all kubelet pod mounts to avoid 'Device or resource busy' on rm.
|
|
641
|
-
shellExec(
|
|
642
|
-
`sudo sh -c 'findmnt --raw --noheadings -o TARGET | grep /var/lib/kubelet | sort -r | xargs -r umount -l'`,
|
|
643
|
-
{ silentOnError: true },
|
|
644
|
-
);
|
|
871
|
+
}
|
|
872
|
+
} catch (error) {
|
|
873
|
+
logger.error(` -> Failed cleaning hostPath PVs: ${error.message}`);
|
|
874
|
+
}
|
|
875
|
+
},
|
|
645
876
|
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
877
|
+
/**
|
|
878
|
+
* @method _lazyUmountKubeletMounts
|
|
879
|
+
* @description Lazy-unmounts every mount under /var/lib/kubelet so a
|
|
880
|
+
* subsequent `rm -rf` does not hit 'Device or resource busy'. Best-effort.
|
|
881
|
+
* @private
|
|
882
|
+
*/
|
|
883
|
+
_lazyUmountKubeletMounts() {
|
|
884
|
+
shellExec(
|
|
885
|
+
`sudo sh -c 'findmnt --raw --noheadings -o TARGET | grep /var/lib/kubelet | sort -r | xargs -r umount -l'`,
|
|
886
|
+
{ silentOnError: true },
|
|
887
|
+
);
|
|
888
|
+
},
|
|
889
|
+
|
|
890
|
+
// Per-type reset methods. Each only touches what its own runtime owns.
|
|
891
|
+
|
|
892
|
+
/**
|
|
893
|
+
* @method safeResetKind
|
|
894
|
+
* @description Kind (Kubernetes in Docker) reset — Docker-scoped only.
|
|
895
|
+
* Does not touch host kubelet / containerd / iptables / SELinux.
|
|
896
|
+
* @param {object} [options]
|
|
897
|
+
* @param {string} [options.underpostRoot='.']
|
|
898
|
+
* @param {boolean} [options.removeVolumeHostPaths=false]
|
|
899
|
+
* @memberof UnderpostCluster
|
|
900
|
+
*/
|
|
901
|
+
async safeResetKind(options = { underpostRoot: '.', removeVolumeHostPaths: false }) {
|
|
902
|
+
logger.info('=== KIND SAFE RESET (development) ===');
|
|
903
|
+
|
|
904
|
+
logger.info('Phase 1/5: Cleaning Kind node-local MongoDB hostPath directories...');
|
|
905
|
+
Underpost.cluster.cleanKindMongoHostPaths({ basePath: '/data/mongodb', replicaCount: 3 });
|
|
906
|
+
|
|
907
|
+
logger.info('Phase 2/5: PersistentVolume hostPath cleanup...');
|
|
908
|
+
if (options.removeVolumeHostPaths) Underpost.cluster._cleanHostPathPvs();
|
|
909
|
+
else logger.info(' -> Skipping (pass --remove-volume-host-paths to enable).');
|
|
910
|
+
|
|
911
|
+
logger.info('Phase 3/5: Deleting all Kind clusters...');
|
|
912
|
+
shellExec(`clusters=$(kind get clusters)
|
|
669
913
|
if [ -n "$clusters" ]; then
|
|
670
914
|
for c in $clusters; do
|
|
671
915
|
echo "Deleting cluster: $c"
|
|
672
916
|
kind delete cluster --name "$c"
|
|
673
917
|
done
|
|
674
918
|
fi`);
|
|
675
|
-
}
|
|
676
919
|
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
920
|
+
logger.info('Phase 4/5: Cleaning kubeconfig and Kind Docker networks...');
|
|
921
|
+
shellExec(`rm -rf "$HOME/.kube"`);
|
|
922
|
+
Underpost.cluster.recoverKindDockerNetworks();
|
|
923
|
+
|
|
924
|
+
logger.info('Phase 5/5: Re-applying host configuration (Docker, containerd, sysctl).');
|
|
925
|
+
Underpost.cluster.config();
|
|
926
|
+
|
|
927
|
+
logger.info('=== KIND SAFE RESET COMPLETE ===');
|
|
928
|
+
},
|
|
929
|
+
|
|
930
|
+
/**
|
|
931
|
+
* @method safeResetKubeadm
|
|
932
|
+
* @description Kubeadm reset on the host: stop kubelet + runtime, kill
|
|
933
|
+
* control-plane ports (6443 / 2379 / 2380 / 10257 / 10259), run
|
|
934
|
+
* `kubeadm reset --force`, wipe kubeadm-managed FS + network state.
|
|
935
|
+
* Does not touch K3s or Kind state.
|
|
936
|
+
* @param {object} [options]
|
|
937
|
+
* @param {string} [options.underpostRoot='.']
|
|
938
|
+
* @param {boolean} [options.removeVolumeHostPaths=false]
|
|
939
|
+
* @memberof UnderpostCluster
|
|
940
|
+
*/
|
|
941
|
+
async safeResetKubeadm(options = { underpostRoot: '.', removeVolumeHostPaths: false }) {
|
|
942
|
+
logger.info('=== KUBEADM SAFE RESET ===');
|
|
943
|
+
|
|
944
|
+
logger.info('Phase 0/7: Truncating large /var/log files...');
|
|
945
|
+
Underpost.cluster._truncateLargeLogs();
|
|
946
|
+
|
|
947
|
+
logger.info('Phase 1/7: PersistentVolume hostPath cleanup...');
|
|
948
|
+
if (options.removeVolumeHostPaths) Underpost.cluster._cleanHostPathPvs();
|
|
949
|
+
else logger.info(' -> Skipping (pass --remove-volume-host-paths to enable).');
|
|
950
|
+
|
|
951
|
+
logger.info('Phase 2/7: SELinux permissive + restore contexts (when present)...');
|
|
952
|
+
shellExec(`if command -v setenforce >/dev/null 2>&1; then sudo setenforce 0; fi`);
|
|
953
|
+
shellExec(
|
|
954
|
+
`if [ -f /etc/selinux/config ]; then sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config; fi`,
|
|
955
|
+
);
|
|
956
|
+
shellExec(
|
|
957
|
+
`if command -v restorecon >/dev/null 2>&1 && [ -d /var/lib/kubelet ]; then sudo restorecon -Rv /var/lib/kubelet; fi`,
|
|
958
|
+
);
|
|
959
|
+
shellExec(
|
|
960
|
+
`if command -v restorecon >/dev/null 2>&1 && [ -d /var/lib/containerd ]; then sudo restorecon -Rv /var/lib/containerd; fi`,
|
|
961
|
+
);
|
|
962
|
+
|
|
963
|
+
logger.info('Phase 3/7: Stopping host kubelet and container runtimes (kubeadm-scope only)...');
|
|
964
|
+
shellExec(`if systemctl is-active --quiet kubelet; then sudo systemctl stop kubelet; fi`);
|
|
965
|
+
shellExec(`if systemctl is-active --quiet docker; then sudo systemctl stop docker; fi`);
|
|
966
|
+
shellExec(`if systemctl is-active --quiet crio; then sudo systemctl stop crio; fi`);
|
|
967
|
+
Underpost.cluster._lazyUmountKubeletMounts();
|
|
968
|
+
|
|
969
|
+
logger.info('Phase 4/7: Killing control-plane processes and running kubeadm reset...');
|
|
970
|
+
shellExec(`if command -v crictl >/dev/null 2>&1; then sudo crictl rm -a -f; fi`, { silentOnError: true });
|
|
971
|
+
// Remove CNI config before stopping sandboxes so Calico's CNI delete hook is
|
|
972
|
+
// not invoked (the API server is already down and the hook would fail).
|
|
973
|
+
shellExec(`sudo rm -rf /etc/cni/net.d/*`);
|
|
974
|
+
shellExec(`if command -v crictl >/dev/null 2>&1; then sudo crictl rmp -a -f; fi`, { silentOnError: true });
|
|
975
|
+
shellExec(`if systemctl is-active --quiet etcd; then sudo systemctl stop etcd; fi`);
|
|
976
|
+
for (const port of [6443, 10259, 10257, 2379, 2380]) {
|
|
977
|
+
shellExec(`if sudo fuser ${port}/tcp >/dev/null 2>&1; then sudo fuser -k ${port}/tcp; fi`);
|
|
718
978
|
}
|
|
979
|
+
shellExec(`if command -v kubeadm >/dev/null 2>&1; then sudo kubeadm reset --force; fi`);
|
|
980
|
+
|
|
981
|
+
logger.info('Phase 5/7: Filesystem cleanup (kubeadm-managed paths only)...');
|
|
982
|
+
shellExec(`sudo rm -rf /etc/kubernetes/*`);
|
|
983
|
+
Underpost.cluster._lazyUmountKubeletMounts();
|
|
984
|
+
shellExec(`sudo rm -rf /var/lib/kubelet/*`);
|
|
985
|
+
shellExec(`sudo rm -rf /var/lib/etcd`);
|
|
986
|
+
shellExec(`sudo rm -rf /var/lib/cni/*`);
|
|
987
|
+
shellExec(`sudo rm -rf /var/lib/containerd/*`);
|
|
988
|
+
shellExec(`rm -rf "$HOME/.kube"`);
|
|
989
|
+
|
|
990
|
+
logger.info('Phase 6/7: Network cleanup (Calico interfaces + host iptables)...');
|
|
991
|
+
shellExec(`if ip link show cni0 >/dev/null 2>&1; then sudo ip link del cni0; fi`);
|
|
992
|
+
shellExec(`if ip link show vxlan.calico >/dev/null 2>&1; then sudo ip link del vxlan.calico; fi`);
|
|
993
|
+
shellExec(`if ip link show tunl0 >/dev/null 2>&1; then sudo ip link del tunl0; fi`);
|
|
994
|
+
shellExec(`sudo iptables -F`);
|
|
995
|
+
shellExec(`sudo iptables -t nat -F`);
|
|
996
|
+
shellExec(`if command -v crictl >/dev/null 2>&1; then sudo crictl rmi --prune; fi`);
|
|
997
|
+
|
|
998
|
+
logger.info('Phase 7/7: Re-applying host configuration (Docker, containerd, sysctl).');
|
|
999
|
+
Underpost.cluster.config();
|
|
1000
|
+
|
|
1001
|
+
logger.info('=== KUBEADM SAFE RESET COMPLETE ===');
|
|
1002
|
+
},
|
|
1003
|
+
|
|
1004
|
+
/**
|
|
1005
|
+
* @method safeResetK3s
|
|
1006
|
+
* @description Centralized K3s teardown. Runs the same way on a physical
|
|
1007
|
+
* host (`node bin cluster --dev --reset --k3s`) or inside an LXD VM via
|
|
1008
|
+
* `lxc exec` (driven by `_resetK3sInVm` in src/cli/lxd.js).
|
|
1009
|
+
* @param {object} [options]
|
|
1010
|
+
* @param {string} [options.underpostRoot='.']
|
|
1011
|
+
* @param {'drain'|'full'} [options.resetMode='full'] - `drain` stops
|
|
1012
|
+
* services + runs `k3s-killall.sh` (K3s persists, returns on next boot).
|
|
1013
|
+
* `full` also runs `k3s-uninstall.sh` and cleans residual state.
|
|
1014
|
+
* @memberof UnderpostCluster
|
|
1015
|
+
*/
|
|
1016
|
+
async safeResetK3s(options = { underpostRoot: '.', resetMode: 'full' }) {
|
|
1017
|
+
const resetMode = options.resetMode === 'drain' ? 'drain' : 'full';
|
|
1018
|
+
logger.info(`=== K3s SAFE RESET (resetMode=${resetMode}) ===`);
|
|
1019
|
+
|
|
1020
|
+
logger.info('Phase 1/5: Stopping K3s systemd units...');
|
|
1021
|
+
shellExec(`if systemctl list-unit-files | grep -q '^k3s\\.service'; then sudo systemctl stop k3s; fi`);
|
|
1022
|
+
shellExec(
|
|
1023
|
+
`if systemctl list-unit-files | grep -q '^k3s-agent\\.service'; then sudo systemctl stop k3s-agent; fi`,
|
|
1024
|
+
);
|
|
1025
|
+
|
|
1026
|
+
logger.info('Phase 2/5: Running k3s-killall.sh (unmount pod overlays, tear down CNI)...');
|
|
1027
|
+
shellExec(`if [ -x /usr/local/bin/k3s-killall.sh ]; then sudo /usr/local/bin/k3s-killall.sh; fi`);
|
|
1028
|
+
|
|
1029
|
+
if (resetMode === 'drain') {
|
|
1030
|
+
logger.info('=== K3s DRAIN COMPLETE (K3s remains installed; will start on next boot) ===');
|
|
1031
|
+
return;
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
logger.info('Phase 3/5: Running k3s-uninstall.sh...');
|
|
1035
|
+
shellExec(`if [ -x /usr/local/bin/k3s-uninstall.sh ]; then sudo /usr/local/bin/k3s-uninstall.sh; fi`);
|
|
1036
|
+
shellExec(`if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then sudo /usr/local/bin/k3s-agent-uninstall.sh; fi`);
|
|
1037
|
+
|
|
1038
|
+
logger.info('Phase 4/5: Removing residual K3s state...');
|
|
1039
|
+
shellExec(`rm -rf "$HOME/.kube"`);
|
|
1040
|
+
shellExec(`if [ -d /etc/rancher/k3s ]; then sudo rm -rf /etc/rancher/k3s; fi`);
|
|
1041
|
+
shellExec(`if ip link show flannel.1 >/dev/null 2>&1; then sudo ip link del flannel.1; fi`);
|
|
1042
|
+
shellExec(`if ip link show cni0 >/dev/null 2>&1; then sudo ip link del cni0; fi`);
|
|
1043
|
+
|
|
1044
|
+
logger.info('Phase 5/5: Re-applying minimal K3s host config.');
|
|
1045
|
+
Underpost.cluster.configMinimalK3s();
|
|
1046
|
+
|
|
1047
|
+
logger.info('=== K3s SAFE RESET COMPLETE (full) ===');
|
|
719
1048
|
},
|
|
720
1049
|
|
|
721
1050
|
/**
|
|
@@ -881,14 +1210,14 @@ EOF`);
|
|
|
881
1210
|
},
|
|
882
1211
|
|
|
883
1212
|
/**
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
1213
|
+
* @method cleanKindMongoHostPaths
|
|
1214
|
+
* @description Best-effort cleanup of MongoDB hostPath directories inside Kind node containers.
|
|
1215
|
+
* This prevents stale replica/auth state when hostPath data lives in node-local container filesystems.
|
|
1216
|
+
* @param {object} [options]
|
|
1217
|
+
* @param {string} [options.basePath='/data/mongodb'] - Node-internal base path for MongoDB data.
|
|
1218
|
+
* @param {number} [options.replicaCount=3] - Number of replica ordinal directories (v0..vN-1).
|
|
1219
|
+
* @memberof UnderpostCluster
|
|
1220
|
+
*/
|
|
892
1221
|
cleanKindMongoHostPaths(options = { basePath: '/data/mongodb', replicaCount: 3 }) {
|
|
893
1222
|
const basePath = options.basePath || '/data/mongodb';
|
|
894
1223
|
const replicaCount = Math.max(Number(options.replicaCount) || 3, 1);
|
|
@@ -919,10 +1248,9 @@ EOF`);
|
|
|
919
1248
|
{ length: replicaCount },
|
|
920
1249
|
(_, index) => `test -d ${basePath}/v${index};`,
|
|
921
1250
|
).join(' ');
|
|
922
|
-
shellExec(
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
);
|
|
1251
|
+
shellExec(`sudo docker exec ${node} sh -lc 'mkdir -p ${basePath}; ${prepareReplicaDirsCmd}'`, {
|
|
1252
|
+
silentOnError: true,
|
|
1253
|
+
});
|
|
926
1254
|
shellExec(`sudo docker exec ${node} sh -lc '${verifyReplicaDirsCmd}'`);
|
|
927
1255
|
}
|
|
928
1256
|
},
|
|
@@ -967,7 +1295,6 @@ EOF`);
|
|
|
967
1295
|
shellExec('sudo systemctl restart docker');
|
|
968
1296
|
shellExec('sudo docker network prune -f', { silentOnError: true });
|
|
969
1297
|
},
|
|
970
|
-
|
|
971
1298
|
};
|
|
972
1299
|
}
|
|
973
1300
|
export default UnderpostCluster;
|