underpost 3.2.90 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/.github/workflows/ghpkg.ci.yml +7 -1
  2. package/.github/workflows/pwa-microservices-template-page.cd.yml +1 -16
  3. package/.github/workflows/pwa-microservices-template-test.ci.yml +1 -1
  4. package/.github/workflows/release.cd.yml +1 -9
  5. package/CHANGELOG.md +110 -1
  6. package/CLI-HELP.md +139 -9
  7. package/README.md +5 -2
  8. package/bin/build.js +7 -5
  9. package/bin/deploy.js +1 -1
  10. package/deploy/lib/logging.sh +96 -0
  11. package/deploy/pwa-microservices-template/deploy.sh +72 -0
  12. package/deploy/release/deploy.sh +62 -0
  13. package/docker-compose.yml +1 -1
  14. package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +5 -1
  15. package/manifests/cronjobs/dd-cron/dd-cron-dns.yaml +1 -1
  16. package/manifests/cronjobs/dd-cron/dd-cron-vultr.yaml +52 -0
  17. package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
  18. package/package.json +5 -5
  19. package/scripts/audit-selinux.sh +64 -0
  20. package/scripts/coverall-test.sh +24 -0
  21. package/scripts/gpu-diag.sh +0 -0
  22. package/scripts/ip-info.sh +0 -0
  23. package/scripts/k3s-node-setup.sh +18 -15
  24. package/scripts/kubeadm-node-setup.sh +12 -23
  25. package/scripts/link-local-underpost-cli.sh +0 -0
  26. package/scripts/lxd-vm-setup.sh +0 -0
  27. package/scripts/maas-nat-firewalld.sh +0 -0
  28. package/scripts/nat-iptables.sh +2 -0
  29. package/scripts/rhel-grpc-setup.sh +0 -0
  30. package/scripts/rocky-kickstart.sh +25 -9
  31. package/scripts/test-monitor.sh +1 -1
  32. package/src/cli/baremetal.js +1 -2
  33. package/src/cli/cloud-init.js +1 -1
  34. package/src/cli/cluster.js +73 -68
  35. package/src/cli/db.js +9 -2
  36. package/src/cli/deploy.js +21 -5
  37. package/src/cli/docker-compose.js +1 -1
  38. package/src/cli/env.js +1 -1
  39. package/src/cli/image.js +0 -1
  40. package/src/cli/index.js +121 -9
  41. package/src/cli/lxd.js +1 -1
  42. package/src/cli/monitor.js +1 -1
  43. package/src/cli/release.js +57 -22
  44. package/src/cli/repository.js +11 -9
  45. package/src/cli/run.js +36 -9
  46. package/src/cli/ssh.js +198 -77
  47. package/src/cli/system.js +26 -13
  48. package/src/cli/test.js +1 -1
  49. package/src/cli/vultr.js +583 -0
  50. package/src/cli/wireguard.js +2125 -0
  51. package/src/client-builder/client-build.js +20 -14
  52. package/src/db/mongo/MongooseDB.js +4 -0
  53. package/src/index.js +25 -1
  54. package/src/projects/underpost/catalog-underpost.js +4 -1
  55. package/src/server/backup.js +1 -1
  56. package/src/server/conf.js +18 -108
  57. package/src/server/cron.js +249 -51
  58. package/src/server/dns.js +100 -6
  59. package/src/server/environment.js +98 -0
  60. package/src/server/forward-proxy.js +549 -0
  61. package/src/server/middlewares.js +56 -1
  62. package/src/server/process.js +0 -1
  63. package/src/server/selinux.js +185 -0
  64. package/src/server/systemd.js +205 -0
  65. package/src/server/underpost-compression.js +186 -0
  66. package/src/server/underpost-gateway.js +20 -10
  67. package/src/server/underpost-ingress.js +18 -2
  68. package/test/selinux.test.js +71 -0
  69. package/test/underpost-gateway.test.js +41 -0
  70. package/test/underpost-ingress.test.js +52 -0
  71. package/test/wireguard-edge.test.js +1177 -0
@@ -0,0 +1,583 @@
1
+ /**
2
+ * Vultr bandwidth guard for the edge hub.
3
+ *
4
+ * The edge VPS is the only machine in the topology that pays for traffic: every
5
+ * byte a client receives leaves through it, and Vultr bills the overage per GB
6
+ * once the plan's monthly transfer quota is spent. Nothing in the request path
7
+ * knows how much of that quota is left — HAProxy forwards bytes it never
8
+ * counts, and the spokes behind the tunnel cannot see the meter at all.
9
+ *
10
+ * This module is that meter. It reads the instance's consumption from the Vultr
11
+ * API, compares it against the plan's quota, and — once a configured fraction of
12
+ * it is gone — reaches the VPS over SSH and drops its egress with
13
+ * {@link module:src/server/dns.js}'s `blockAllEgress`.
14
+ *
15
+ * That last step is deliberately blunt: it takes every hostname behind the hub
16
+ * offline. It is the cheaper failure. An overage accrues silently and without a
17
+ * ceiling, while a blocked edge is loud, immediate, and reversible with one
18
+ * command. `blockAllEgress` keeps established and related connections, so a new
19
+ * inbound SSH session still completes its handshake and the host stays
20
+ * reachable to undo it.
21
+ *
22
+ * The enforcement is latched in the root env rather than re-applied every run,
23
+ * so a cron firing every ten minutes does not re-open an SSH session to a host
24
+ * that is already blocked.
25
+ *
26
+ * @module src/cli/vultr.js
27
+ * @namespace UnderpostVultr
28
+ */
29
+
30
+ import axios from 'axios';
31
+ import { environmentValueFactory } from '../server/environment.js';
32
+ import { FORWARD_PROXY, fetchViaForwardProxy } from '../server/forward-proxy.js';
33
+ import { loggerFactory } from '../server/logger.js';
34
+ import Underpost from '../index.js';
35
+
36
+ const logger = loggerFactory(import.meta);
37
+
38
+ /**
39
+ * @constant UNDERPOST_VULTR
40
+ * @description Fixed identity of the bandwidth guard: API surface, defaults, and
41
+ * the key the enforcement state is latched under.
42
+ * @memberof UnderpostVultr
43
+ */
44
+ const UNDERPOST_VULTR = {
45
+ apiBaseUrl: 'https://api.vultr.com/v2',
46
+ // Vultr quotes plan `bandwidth` in GB and usage in bytes, so one of the two
47
+ // has to be converted. GB here is binary, matching how the quota is metered.
48
+ bytesPerGB: 1024 * 1024 * 1024,
49
+ defaultThreshold: 0.8,
50
+ // The documented maximum; fewer pages means fewer round trips before the
51
+ // instance's plan is found.
52
+ plansPerPage: 500,
53
+ // A cursor loop bounded so a malformed `meta.links.next` cannot spin forever.
54
+ maxPlanPages: 20,
55
+ requestTimeoutMs: 20000,
56
+ defaultSshUser: 'root',
57
+ defaultSshPort: 22,
58
+ defaultSshKeyPath: './engine-private/deploy/id_rsa',
59
+ remoteEnginePath: '/home/dd/engine',
60
+ // Latched in the root env, which the CronJob mounts from the host, so the
61
+ // decision survives the container that made it.
62
+ latchKey: 'VULTR_EGRESS_BLOCKED_AT',
63
+ env: {
64
+ apiKey: 'VULTR_API_KEY',
65
+ instanceId: 'VULTR_INSTANCE_ID',
66
+ threshold: 'VULTR_BANDWIDTH_THRESHOLD',
67
+ host: ['VULTR_VPS_IP', 'DEFAULT_SSH_HOST'],
68
+ user: ['VULTR_SSH_USER', 'DEFAULT_SSH_USER'],
69
+ keyPath: ['VULTR_SSH_KEY_PATH', 'DEFAULT_SSH_KEY_PATH'],
70
+ port: ['VULTR_SSH_PORT', 'DEFAULT_SSH_PORT'],
71
+ // The edge hub's forward proxy, named once in its canonical module so the two
72
+ // ends of it cannot disagree about the variable that configures it.
73
+ forwardProxyApiKey: FORWARD_PROXY.env.apiKey,
74
+ forwardProxyHost: FORWARD_PROXY.env.host,
75
+ forwardProxyPort: FORWARD_PROXY.env.port,
76
+ },
77
+ };
78
+
79
+ /**
80
+ * @method envFactory
81
+ * @description First non-empty value among a list of keys.
82
+ *
83
+ * Resolution is {@link module:src/server/environment.js.environmentValueFactory}'s — the process
84
+ * environment, then the deploy env `underpost env <deploy-id> <environment>`
85
+ * selects into `./.env`, then the underpost root env — rather than a second
86
+ * implementation of it, because the three callers differ: a CronJob container has
87
+ * its deploy env loaded into `process.env` by `loadCronDeployEnv`, an operator
88
+ * preparing a manual run has `./.env`, and an operator who ran `underpost env
89
+ * set` has the root env. Never logged — one of the keys this resolves is an API
90
+ * key.
91
+ * @param {string|Array<string>} keys - Environment variable name, or names in precedence order.
92
+ * @returns {string} The resolved value, or an empty string.
93
+ * @memberof UnderpostVultr
94
+ */
95
+ const envFactory = (keys) => {
96
+ for (const key of Array.isArray(keys) ? keys : [keys]) {
97
+ const value = environmentValueFactory(key);
98
+ if (value) return value;
99
+ }
100
+ return '';
101
+ };
102
+
103
+ /**
104
+ * @method thresholdFactory
105
+ * @description Normalizes the configured trigger fraction.
106
+ *
107
+ * `80` and `0.80` are the same intent expressed two ways, and the first one
108
+ * silently never fires — a guard that never fires is worse than no guard, since
109
+ * it reads as protection. Both are accepted and resolve to the same ratio.
110
+ * @param {string|number} [raw] - Configured value.
111
+ * @param {number} [fallback] - Ratio used when nothing usable is configured.
112
+ * @returns {number} Ratio in `(0, 1]`.
113
+ * @memberof UnderpostVultr
114
+ */
115
+ const thresholdFactory = (raw, fallback = UNDERPOST_VULTR.defaultThreshold) => {
116
+ const parsed = parseFloat(`${raw ?? ''}`.trim());
117
+ if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
118
+ const ratio = parsed > 1 ? parsed / 100 : parsed;
119
+ return ratio > 1 ? 1 : ratio;
120
+ };
121
+
122
+ /**
123
+ * @method billingMonthFactory
124
+ * @description The `YYYY-MM` prefix the daily buckets of the current cycle carry.
125
+ *
126
+ * UTC, because the endpoint's day boundaries are UTC. A host in a negative
127
+ * offset would otherwise drop the current day for part of its evening.
128
+ * @param {Date} [now] - Reference instant.
129
+ * @returns {string} `YYYY-MM`.
130
+ * @memberof UnderpostVultr
131
+ */
132
+ const billingMonthFactory = (now = new Date()) =>
133
+ `${now.getUTCFullYear()}-${`${now.getUTCMonth() + 1}`.padStart(2, '0')}`;
134
+
135
+ /**
136
+ * @method bandwidthTotalsFactory
137
+ * @description Folds the endpoint's daily buckets into one consumption figure.
138
+ *
139
+ * Scoped to the current month by default rather than summing every key
140
+ * returned. The response is a rolling window that can still carry the tail of
141
+ * the previous cycle, and those bytes are against a quota that has already
142
+ * reset — counting them reports a host as over budget on the first days of a
143
+ * month when it has barely spent anything.
144
+ *
145
+ * Both directions are returned separately because which of them is billable is
146
+ * a property of the account's plan, not of this code: `total` is the
147
+ * conservative reading and trips first, `outgoing` is the one that maps to
148
+ * egress alone.
149
+ * @param {object} [bandwidth] - `bandwidth` object from the Vultr response.
150
+ * @param {string} [month] - `YYYY-MM` to scope to; empty sums every bucket.
151
+ * @returns {{totalBytes: number, incomingBytes: number, outgoingBytes: number, days: number, dates: Array<string>}} Consumption for the window.
152
+ * @memberof UnderpostVultr
153
+ */
154
+ const bandwidthTotalsFactory = ({ bandwidth = {}, month = billingMonthFactory() } = {}) => {
155
+ const dates = Object.keys(bandwidth || {})
156
+ .filter((date) => !month || `${date}`.startsWith(month))
157
+ .sort();
158
+ let incomingBytes = 0;
159
+ let outgoingBytes = 0;
160
+ for (const date of dates) {
161
+ const bucket = bandwidth[date] || {};
162
+ incomingBytes += Number(bucket.incoming_bytes) || 0;
163
+ outgoingBytes += Number(bucket.outgoing_bytes) || 0;
164
+ }
165
+ return { totalBytes: incomingBytes + outgoingBytes, incomingBytes, outgoingBytes, days: dates.length, dates };
166
+ };
167
+
168
+ /**
169
+ * @method quotaStateFactory
170
+ * @description The consumption decision: quota, effective trigger, and whether
171
+ * it has been crossed.
172
+ *
173
+ * A plan with no quota (`bandwidth: 0`, which Vultr uses for unmetered plans)
174
+ * yields `exceeded: false` and is reported as such rather than dividing by zero
175
+ * and blocking a host that cannot run out.
176
+ * @param {number} consumedBytes - Bytes counted for the cycle.
177
+ * @param {number} planBandwidthGB - Plan quota in GB.
178
+ * @param {number} [threshold] - Trigger fraction of the quota.
179
+ * @returns {{maxBytes: number, limitBytes: number, consumedBytes: number, ratio: number, exceeded: boolean, metered: boolean}} Decision inputs and outcome.
180
+ * @memberof UnderpostVultr
181
+ */
182
+ const quotaStateFactory = ({
183
+ consumedBytes = 0,
184
+ planBandwidthGB = 0,
185
+ threshold = UNDERPOST_VULTR.defaultThreshold,
186
+ }) => {
187
+ const maxBytes = Math.max(0, Number(planBandwidthGB) || 0) * UNDERPOST_VULTR.bytesPerGB;
188
+ const limitBytes = maxBytes * threshold;
189
+ const metered = maxBytes > 0;
190
+ return {
191
+ maxBytes,
192
+ limitBytes,
193
+ consumedBytes,
194
+ ratio: metered ? consumedBytes / maxBytes : 0,
195
+ exceeded: metered && consumedBytes >= limitBytes,
196
+ metered,
197
+ };
198
+ };
199
+
200
+ /**
201
+ * @method formatBytes
202
+ * @description Byte count as GB, for log lines an operator has to read against
203
+ * a plan quota that is quoted in GB.
204
+ * @param {number} bytes - Byte count.
205
+ * @returns {string} e.g. `812.40 GB`.
206
+ * @memberof UnderpostVultr
207
+ */
208
+ const formatBytes = (bytes) => `${((Number(bytes) || 0) / UNDERPOST_VULTR.bytesPerGB).toFixed(2)} GB`;
209
+
210
+ /**
211
+ * @method vultrUrlFactory
212
+ * @description One absolute API URL, with its query already encoded.
213
+ *
214
+ * Built here rather than left to the transport because the two transports below
215
+ * take a query differently — axios takes `params`, the proxy client takes a URL —
216
+ * and a request that differs by transport is a bug waiting for the day the
217
+ * fallback is used.
218
+ * @param {string} path - Path below `/v2`.
219
+ * @param {object} [params] - Query parameters.
220
+ * @returns {string} Absolute URL.
221
+ * @memberof UnderpostVultr
222
+ */
223
+ const vultrUrlFactory = ({ path, params = {} }) => {
224
+ const url = new URL(`${UNDERPOST_VULTR.apiBaseUrl}${path}`);
225
+ for (const [key, value] of Object.entries(params))
226
+ if (value !== undefined && value !== null && `${value}` !== '') url.searchParams.set(key, `${value}`);
227
+ return url.href;
228
+ };
229
+
230
+ /**
231
+ * @method vultrGet
232
+ * @description The transport: through the edge hub's forward proxy when one is
233
+ * configured, straight out otherwise.
234
+ *
235
+ * The proxy is the point of this indirection. Vultr's API sees the address the
236
+ * request came from, and this job usually runs in a CronJob inside a homelab
237
+ * cluster — so a direct call arrives from a residential ISP address, while a
238
+ * proxied one arrives from the very VPS the job is metering. An API key scoped to
239
+ * the edge's address only works over the proxy.
240
+ *
241
+ * Non-2xx answers are returned rather than thrown, so both transports report a
242
+ * status the same way; only a transport failure throws.
243
+ * @param {string} url - Absolute URL.
244
+ * @param {string} apiKey - Vultr API key.
245
+ * @param {object} [proxy] - Forward proxy endpoint; ignored when it carries no key.
246
+ * @returns {Promise<{status: number, data: object}>} Status and parsed body.
247
+ * @memberof UnderpostVultr
248
+ */
249
+ const vultrGet = async ({ url, apiKey, proxy }) => {
250
+ const headers = { Authorization: `Bearer ${apiKey}`, Accept: 'application/json' };
251
+ if (!proxy?.apiKey) {
252
+ const response = await axios.get(url, {
253
+ headers,
254
+ timeout: UNDERPOST_VULTR.requestTimeoutMs,
255
+ validateStatus: () => true,
256
+ });
257
+ return { status: response.status, data: response.data };
258
+ }
259
+ const { status, body } = await fetchViaForwardProxy(url, {
260
+ headers,
261
+ timeout: UNDERPOST_VULTR.requestTimeoutMs,
262
+ proxy,
263
+ });
264
+ try {
265
+ return { status, data: body ? JSON.parse(body) : {} };
266
+ } catch {
267
+ return { status, data: {} };
268
+ }
269
+ };
270
+
271
+ /**
272
+ * @method vultrRequest
273
+ * @description One authenticated Vultr API call.
274
+ *
275
+ * Errors are re-thrown with Vultr's own message and without the request config,
276
+ * because an axios error carries the `Authorization` header it was sent with and
277
+ * this module's failures are logged.
278
+ * @param {string} apiKey - Vultr API key.
279
+ * @param {string} path - Path below `/v2`.
280
+ * @param {object} [params] - Query parameters.
281
+ * @param {object} [proxy] - Forward proxy endpoint from {@link UnderpostVultr.resolveConfig}.
282
+ * @returns {Promise<object>} Response body.
283
+ * @throws {Error} With the API's status and message, never the credentials.
284
+ * @memberof UnderpostVultr
285
+ */
286
+ const vultrRequest = async ({ apiKey, path, params = {}, proxy = null }) => {
287
+ let status = 0;
288
+ let data = {};
289
+ try {
290
+ ({ status, data } = await vultrGet({ url: vultrUrlFactory({ path, params }), apiKey, proxy }));
291
+ } catch (error) {
292
+ const responseStatus = error?.response?.status;
293
+ const detail = error?.response?.data?.error || error?.message || 'request failed';
294
+ const code = error?.code || error?.cause?.code;
295
+ const proxyHint =
296
+ proxy?.apiKey && ['EHOSTUNREACH', 'ENETUNREACH', 'ECONNREFUSED', 'ETIMEDOUT'].includes(code)
297
+ ? '; verify wg0 on the spoke; if only pods fail, re-run --wireguard-setup --client and restart wg0; if the host also fails, re-run --forward-proxy-server on the hub'
298
+ : '';
299
+ throw new Error(
300
+ `[vultr] GET /v2${path} failed${responseStatus ? ` (${responseStatus})` : ''}: ${detail}${proxyHint}`,
301
+ );
302
+ }
303
+ if (status < 200 || status >= 300)
304
+ throw new Error(`[vultr] GET /v2${path} failed (${status}): ${data?.error || 'request failed'}`);
305
+ return data;
306
+ };
307
+
308
+ /**
309
+ * @method planBandwidthGBFactory
310
+ * @description The monthly transfer quota, in GB, of the plan an instance runs.
311
+ *
312
+ * `/v2/plans` is paginated and the catalogue is longer than one page, so the
313
+ * cursor is followed rather than the first page searched — a plan that happens
314
+ * to sort past the page boundary would otherwise read as "not found" and, on a
315
+ * naive implementation, as a quota of zero.
316
+ * @param {string} apiKey - Vultr API key.
317
+ * @param {string} planId - Plan id from the instance record.
318
+ * @param {object} [proxy] - Forward proxy endpoint.
319
+ * @returns {Promise<number>} Quota in GB.
320
+ * @throws {Error} When the plan is absent from the catalogue.
321
+ * @memberof UnderpostVultr
322
+ */
323
+ const planBandwidthGBFactory = async ({ apiKey, planId, proxy = null }) => {
324
+ let cursor = '';
325
+ for (let page = 0; page < UNDERPOST_VULTR.maxPlanPages; page++) {
326
+ const data = await vultrRequest({
327
+ apiKey,
328
+ path: '/plans',
329
+ params: { type: 'all', per_page: UNDERPOST_VULTR.plansPerPage, ...(cursor ? { cursor } : {}) },
330
+ proxy,
331
+ });
332
+ const match = (data?.plans || []).find((plan) => plan?.id === planId);
333
+ if (match) return Number(match.bandwidth) || 0;
334
+ cursor = `${data?.meta?.links?.next || ''}`.trim();
335
+ if (!cursor) break;
336
+ }
337
+ throw new Error(`[vultr] Plan ${planId} was not found in the plan catalogue; cannot resolve its bandwidth quota`);
338
+ };
339
+
340
+ /**
341
+ * @class UnderpostVultr
342
+ * @description Bandwidth accounting for the edge VPS, and the egress cut-off it
343
+ * triggers.
344
+ * @memberof UnderpostVultr
345
+ */
346
+ class UnderpostVultr {
347
+ static API = {
348
+ /**
349
+ * @method callback
350
+ * @description CLI and cron entry point.
351
+ *
352
+ * Takes the same `(deployList, options)` shape every other
353
+ * {@link UnderpostCron} job does, so `underpost cron default vultr` dispatches
354
+ * to it unchanged. The deploy list is not used to select an instance — the
355
+ * edge hub is one machine for the whole cluster, exactly as its WireGuard
356
+ * peer registry is — but it is logged so a run is attributable.
357
+ * @param {string} [deployList] - Comma-separated deploy ids, from the cron dispatcher.
358
+ * @param {object} [options] - CLI flags.
359
+ * @returns {Promise<object>} Result from {@link UnderpostVultr.checkBandwidth}.
360
+ * @memberof UnderpostVultr
361
+ */
362
+ callback: async function (deployList = 'default', options = {}) {
363
+ return await UnderpostVultr.API.checkBandwidth({ ...options, deployList });
364
+ },
365
+
366
+ /**
367
+ * @method resolveConfig
368
+ * @description Every input the guard needs, resolved once.
369
+ *
370
+ * CLI flags win over the environment so a one-off run can target another
371
+ * instance without editing any `.env`; the environment is the standing
372
+ * configuration. Single source of truth for the resolution, so no other
373
+ * method re-reads `process.env`.
374
+ * @param {object} [options] - CLI flags.
375
+ * @returns {object} Resolved configuration; `apiKey` is present but must never be logged.
376
+ * @memberof UnderpostVultr
377
+ */
378
+ resolveConfig(options = {}) {
379
+ return {
380
+ apiKey: `${options.apiKey || ''}`.trim() || envFactory(UNDERPOST_VULTR.env.apiKey),
381
+ instanceId: `${options.instanceId || ''}`.trim() || envFactory(UNDERPOST_VULTR.env.instanceId),
382
+ threshold: thresholdFactory(options.threshold || envFactory(UNDERPOST_VULTR.env.threshold)),
383
+ host: `${options.host || ''}`.trim() || envFactory(UNDERPOST_VULTR.env.host),
384
+ user: `${options.user || ''}`.trim() || envFactory(UNDERPOST_VULTR.env.user) || UNDERPOST_VULTR.defaultSshUser,
385
+ keyPath:
386
+ `${options.keyPath || ''}`.trim() ||
387
+ envFactory(UNDERPOST_VULTR.env.keyPath) ||
388
+ UNDERPOST_VULTR.defaultSshKeyPath,
389
+ port: Number(options.port || envFactory(UNDERPOST_VULTR.env.port)) || UNDERPOST_VULTR.defaultSshPort,
390
+ // `total` is the conservative reading and trips first; `outgoing` counts
391
+ // egress alone, which is what a plan billing outbound-only meters.
392
+ metric: `${options.metric || 'total'}`.trim() === 'outgoing' ? 'outgoing' : 'total',
393
+ month: options.allDates === true ? '' : `${options.month || ''}`.trim() || billingMonthFactory(),
394
+ dryRun: options.dryRun === true,
395
+ force: options.force === true,
396
+ autoUnblock: options.autoUnblock === true,
397
+ // Resolved through the same env precedence as everything else, so the
398
+ // cron's deploy env can enable the proxy without a flag. An unset key
399
+ // means no proxy, and the API is called directly.
400
+ forwardProxy: {
401
+ apiKey: envFactory(UNDERPOST_VULTR.env.forwardProxyApiKey),
402
+ host: envFactory(UNDERPOST_VULTR.env.forwardProxyHost),
403
+ port: envFactory(UNDERPOST_VULTR.env.forwardProxyPort),
404
+ },
405
+ };
406
+ },
407
+
408
+ /**
409
+ * @method checkBandwidth
410
+ * @description Reads the instance's consumption for the cycle and enforces
411
+ * the threshold.
412
+ *
413
+ * Three calls, in the only order that works: the instance record names its
414
+ * plan, the plan carries the quota, and the bandwidth endpoint carries the
415
+ * consumption. Nothing is enforced until all three have answered — a guard
416
+ * that blocked an edge on a failed API call would be an outage caused by the
417
+ * thing meant to prevent one.
418
+ * @param {object} [options] - CLI flags; see {@link UnderpostVultr.resolveConfig}.
419
+ * @returns {Promise<{instanceId: string, plan: string, metric: string, month: string, consumedBytes: number, limitBytes: number, maxBytes: number, ratio: number, exceeded: boolean, enforced: boolean, latched: boolean}>} What was measured and what was done about it.
420
+ * @throws {Error} When credentials are missing or the API cannot be read.
421
+ * @memberof UnderpostVultr
422
+ */
423
+ checkBandwidth: async function (options = {}) {
424
+ const config = UnderpostVultr.API.resolveConfig(options);
425
+ if (!config.apiKey) throw new Error(`[vultr] ${UNDERPOST_VULTR.env.apiKey} is not set`);
426
+ if (!config.instanceId) throw new Error(`[vultr] ${UNDERPOST_VULTR.env.instanceId} is not set`);
427
+
428
+ const proxy = config.forwardProxy;
429
+ const instance = (await vultrRequest({ apiKey: config.apiKey, path: `/instances/${config.instanceId}`, proxy }))
430
+ ?.instance;
431
+ const planId = `${instance?.plan || ''}`.trim();
432
+ if (!planId) throw new Error(`[vultr] Instance ${config.instanceId} returned no plan id`);
433
+
434
+ const planBandwidthGB = await planBandwidthGBFactory({ apiKey: config.apiKey, planId, proxy });
435
+ const { bandwidth } = await vultrRequest({
436
+ apiKey: config.apiKey,
437
+ path: `/instances/${config.instanceId}/bandwidth`,
438
+ proxy,
439
+ });
440
+
441
+ const totals = bandwidthTotalsFactory({ bandwidth, month: config.month });
442
+ const consumedBytes = config.metric === 'outgoing' ? totals.outgoingBytes : totals.totalBytes;
443
+ const state = quotaStateFactory({ consumedBytes, planBandwidthGB, threshold: config.threshold });
444
+ const latchedAt = `${Underpost.env.get(UNDERPOST_VULTR.latchKey, undefined, { disableLog: true }) ?? ''}`.trim();
445
+
446
+ const summary = {
447
+ instanceId: config.instanceId,
448
+ plan: planId,
449
+ metric: config.metric,
450
+ month: config.month || 'all-dates',
451
+ days: totals.days,
452
+ consumed: formatBytes(consumedBytes),
453
+ incoming: formatBytes(totals.incomingBytes),
454
+ outgoing: formatBytes(totals.outgoingBytes),
455
+ triggerLimit: formatBytes(state.limitBytes),
456
+ planQuota: formatBytes(state.maxBytes),
457
+ usedPercent: `${(state.ratio * 100).toFixed(1)}%`,
458
+ threshold: `${(config.threshold * 100).toFixed(0)}%`,
459
+ // Which address Vultr saw the reads come from, which is the difference
460
+ // between a key scoped to the edge VPS working and failing.
461
+ via: proxy.apiKey ? `forward-proxy ${proxy.host || FORWARD_PROXY.defaultHost}` : 'direct',
462
+ };
463
+
464
+ if (!state.metered) {
465
+ logger.info('Vultr plan reports no metered bandwidth quota; nothing to enforce', summary);
466
+ return { ...state, ...summary, enforced: false, latched: !!latchedAt };
467
+ }
468
+
469
+ if (!state.exceeded) {
470
+ logger.info('Vultr bandwidth within budget', summary);
471
+ // A latch that outlives the cycle it was set in would suppress the next
472
+ // real trigger, so it is cleared as soon as usage is back under the
473
+ // threshold — but the host stays blocked until someone says otherwise.
474
+ if (latchedAt) {
475
+ if (!config.dryRun) Underpost.env.delete(UNDERPOST_VULTR.latchKey);
476
+ if (config.autoUnblock) await UnderpostVultr.API.setEdgeEgress({ config, blocked: false });
477
+ else
478
+ logger.warn('Edge egress is still blocked from a previous cycle; unblock it when you are ready', {
479
+ host: config.host,
480
+ blockedAt: latchedAt,
481
+ next: `underpost ip --unblock-all-egress (on ${config.host || 'the edge VPS'})`,
482
+ });
483
+ }
484
+ return { ...state, ...summary, enforced: false, latched: false };
485
+ }
486
+
487
+ logger.warn('Vultr bandwidth threshold reached; edge egress will be blocked', summary);
488
+
489
+ if (latchedAt && !config.force) {
490
+ logger.info('Edge egress was already blocked for this cycle; not re-applying', {
491
+ host: config.host,
492
+ blockedAt: latchedAt,
493
+ next: 'pass --force to re-apply',
494
+ });
495
+ return { ...state, ...summary, enforced: false, latched: true };
496
+ }
497
+
498
+ const enforced = await UnderpostVultr.API.setEdgeEgress({ config, blocked: true });
499
+ if (enforced && !config.dryRun) Underpost.env.set(UNDERPOST_VULTR.latchKey, new Date().toISOString());
500
+ return { ...state, ...summary, enforced, latched: enforced };
501
+ },
502
+
503
+ /**
504
+ * @method setEdgeEgress
505
+ * @description Blocks or restores outbound traffic on the edge VPS over SSH.
506
+ *
507
+ * The command is run on the VPS rather than here because the nftables rules
508
+ * belong to that host — this process usually runs in a CronJob container
509
+ * inside a spoke cluster, on the far side of the tunnel the rules govern.
510
+ *
511
+ * `underpost ip` is preferred when the CLI is installed globally and the
512
+ * checked-out engine is the fallback, so a VPS provisioned either way is
513
+ * reachable. The command already elevates internally, so it is not wrapped
514
+ * in `sudo` here.
515
+ * @param {object} config - Resolved configuration from {@link UnderpostVultr.resolveConfig}.
516
+ * @param {boolean} blocked - True to block egress, false to restore it.
517
+ * @returns {Promise<boolean>} True when the remote command succeeded.
518
+ * @memberof UnderpostVultr
519
+ */
520
+ setEdgeEgress: async function ({ config, blocked }) {
521
+ const flag = blocked ? '--block-all-egress' : '--unblock-all-egress';
522
+ if (!config.host) {
523
+ logger.error('No edge host configured; cannot reach the VPS to change its egress', {
524
+ set: UNDERPOST_VULTR.env.host.join(' or '),
525
+ });
526
+ return false;
527
+ }
528
+ const command = [
529
+ 'set -e',
530
+ 'if command -v underpost >/dev/null 2>&1; then',
531
+ ` underpost ip ${flag}`,
532
+ 'else',
533
+ ` cd ${UNDERPOST_VULTR.remoteEnginePath} && node bin ip ${flag}`,
534
+ 'fi',
535
+ ].join('\n');
536
+
537
+ if (config.dryRun) {
538
+ logger.info('[dry-run] would run on the edge VPS', {
539
+ target: `${config.user}@${config.host}:${config.port}`,
540
+ command: `underpost ip ${flag}`,
541
+ });
542
+ return false;
543
+ }
544
+
545
+ const result = await Underpost.ssh.sshExecBatch({
546
+ host: config.host,
547
+ port: config.port,
548
+ user: config.user,
549
+ keyPath: config.keyPath,
550
+ command,
551
+ });
552
+ if (!result.ok) {
553
+ logger.error(`Failed to run 'underpost ip ${flag}' on the edge VPS`, {
554
+ target: `${config.user}@${config.host}:${config.port}`,
555
+ code: result.code,
556
+ stderr: `${result.stderr || ''}`.slice(-400),
557
+ });
558
+ return false;
559
+ }
560
+ if (blocked)
561
+ logger.warn('Edge egress blocked; every hostname behind the hub is now offline', {
562
+ host: config.host,
563
+ restore: `underpost ip --unblock-all-egress (on ${config.host})`,
564
+ });
565
+ else logger.info('Edge egress restored', { host: config.host });
566
+ return true;
567
+ },
568
+ };
569
+ }
570
+
571
+ export default UnderpostVultr;
572
+
573
+ export {
574
+ UNDERPOST_VULTR,
575
+ bandwidthTotalsFactory,
576
+ billingMonthFactory,
577
+ envFactory,
578
+ formatBytes,
579
+ planBandwidthGBFactory,
580
+ quotaStateFactory,
581
+ thresholdFactory,
582
+ vultrUrlFactory,
583
+ };