underpost 3.2.90 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ghpkg.ci.yml +7 -1
- package/.github/workflows/pwa-microservices-template-page.cd.yml +1 -16
- package/.github/workflows/pwa-microservices-template-test.ci.yml +1 -1
- package/.github/workflows/release.cd.yml +1 -9
- package/CHANGELOG.md +110 -1
- package/CLI-HELP.md +139 -9
- package/README.md +5 -2
- package/bin/build.js +7 -5
- package/bin/deploy.js +1 -1
- package/deploy/lib/logging.sh +96 -0
- package/deploy/pwa-microservices-template/deploy.sh +72 -0
- package/deploy/release/deploy.sh +62 -0
- package/docker-compose.yml +1 -1
- package/manifests/cronjobs/dd-cron/dd-cron-backup.yaml +5 -1
- package/manifests/cronjobs/dd-cron/dd-cron-dns.yaml +1 -1
- package/manifests/cronjobs/dd-cron/dd-cron-vultr.yaml +52 -0
- package/manifests/deployment/dd-default-development/deployment.yaml +2 -2
- package/package.json +5 -5
- package/scripts/audit-selinux.sh +64 -0
- package/scripts/coverall-test.sh +24 -0
- package/scripts/gpu-diag.sh +0 -0
- package/scripts/ip-info.sh +0 -0
- package/scripts/k3s-node-setup.sh +18 -15
- package/scripts/kubeadm-node-setup.sh +12 -23
- package/scripts/link-local-underpost-cli.sh +0 -0
- package/scripts/lxd-vm-setup.sh +0 -0
- package/scripts/maas-nat-firewalld.sh +0 -0
- package/scripts/nat-iptables.sh +2 -0
- package/scripts/rhel-grpc-setup.sh +0 -0
- package/scripts/rocky-kickstart.sh +25 -9
- package/scripts/test-monitor.sh +1 -1
- package/src/cli/baremetal.js +1 -2
- package/src/cli/cloud-init.js +1 -1
- package/src/cli/cluster.js +73 -68
- package/src/cli/db.js +9 -2
- package/src/cli/deploy.js +21 -5
- package/src/cli/docker-compose.js +1 -1
- package/src/cli/env.js +1 -1
- package/src/cli/image.js +0 -1
- package/src/cli/index.js +121 -9
- package/src/cli/lxd.js +1 -1
- package/src/cli/monitor.js +1 -1
- package/src/cli/release.js +57 -22
- package/src/cli/repository.js +11 -9
- package/src/cli/run.js +36 -9
- package/src/cli/ssh.js +198 -77
- package/src/cli/system.js +26 -13
- package/src/cli/test.js +1 -1
- package/src/cli/vultr.js +583 -0
- package/src/cli/wireguard.js +2125 -0
- package/src/client-builder/client-build.js +20 -14
- package/src/db/mongo/MongooseDB.js +4 -0
- package/src/index.js +25 -1
- package/src/projects/underpost/catalog-underpost.js +4 -1
- package/src/server/backup.js +1 -1
- package/src/server/conf.js +18 -108
- package/src/server/cron.js +249 -51
- package/src/server/dns.js +100 -6
- package/src/server/environment.js +98 -0
- package/src/server/forward-proxy.js +549 -0
- package/src/server/middlewares.js +56 -1
- package/src/server/process.js +0 -1
- package/src/server/selinux.js +185 -0
- package/src/server/systemd.js +205 -0
- package/src/server/underpost-compression.js +186 -0
- package/src/server/underpost-gateway.js +20 -10
- package/src/server/underpost-ingress.js +18 -2
- package/test/selinux.test.js +71 -0
- package/test/underpost-gateway.test.js +41 -0
- package/test/underpost-ingress.test.js +52 -0
- package/test/wireguard-edge.test.js +1177 -0
package/src/cli/vultr.js
ADDED
|
@@ -0,0 +1,583 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vultr bandwidth guard for the edge hub.
|
|
3
|
+
*
|
|
4
|
+
* The edge VPS is the only machine in the topology that pays for traffic: every
|
|
5
|
+
* byte a client receives leaves through it, and Vultr bills the overage per GB
|
|
6
|
+
* once the plan's monthly transfer quota is spent. Nothing in the request path
|
|
7
|
+
* knows how much of that quota is left — HAProxy forwards bytes it never
|
|
8
|
+
* counts, and the spokes behind the tunnel cannot see the meter at all.
|
|
9
|
+
*
|
|
10
|
+
* This module is that meter. It reads the instance's consumption from the Vultr
|
|
11
|
+
* API, compares it against the plan's quota, and — once a configured fraction of
|
|
12
|
+
* it is gone — reaches the VPS over SSH and drops its egress with
|
|
13
|
+
* {@link module:src/server/dns.js}'s `blockAllEgress`.
|
|
14
|
+
*
|
|
15
|
+
* That last step is deliberately blunt: it takes every hostname behind the hub
|
|
16
|
+
* offline. It is the cheaper failure. An overage accrues silently and without a
|
|
17
|
+
* ceiling, while a blocked edge is loud, immediate, and reversible with one
|
|
18
|
+
* command. `blockAllEgress` keeps established and related connections, so a new
|
|
19
|
+
* inbound SSH session still completes its handshake and the host stays
|
|
20
|
+
* reachable to undo it.
|
|
21
|
+
*
|
|
22
|
+
* The enforcement is latched in the root env rather than re-applied every run,
|
|
23
|
+
* so a cron firing every ten minutes does not re-open an SSH session to a host
|
|
24
|
+
* that is already blocked.
|
|
25
|
+
*
|
|
26
|
+
* @module src/cli/vultr.js
|
|
27
|
+
* @namespace UnderpostVultr
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import axios from 'axios';
|
|
31
|
+
import { environmentValueFactory } from '../server/environment.js';
|
|
32
|
+
import { FORWARD_PROXY, fetchViaForwardProxy } from '../server/forward-proxy.js';
|
|
33
|
+
import { loggerFactory } from '../server/logger.js';
|
|
34
|
+
import Underpost from '../index.js';
|
|
35
|
+
|
|
36
|
+
const logger = loggerFactory(import.meta);
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* @constant UNDERPOST_VULTR
|
|
40
|
+
* @description Fixed identity of the bandwidth guard: API surface, defaults, and
|
|
41
|
+
* the key the enforcement state is latched under.
|
|
42
|
+
* @memberof UnderpostVultr
|
|
43
|
+
*/
|
|
44
|
+
const UNDERPOST_VULTR = {
|
|
45
|
+
apiBaseUrl: 'https://api.vultr.com/v2',
|
|
46
|
+
// Vultr quotes plan `bandwidth` in GB and usage in bytes, so one of the two
|
|
47
|
+
// has to be converted. GB here is binary, matching how the quota is metered.
|
|
48
|
+
bytesPerGB: 1024 * 1024 * 1024,
|
|
49
|
+
defaultThreshold: 0.8,
|
|
50
|
+
// The documented maximum; fewer pages means fewer round trips before the
|
|
51
|
+
// instance's plan is found.
|
|
52
|
+
plansPerPage: 500,
|
|
53
|
+
// A cursor loop bounded so a malformed `meta.links.next` cannot spin forever.
|
|
54
|
+
maxPlanPages: 20,
|
|
55
|
+
requestTimeoutMs: 20000,
|
|
56
|
+
defaultSshUser: 'root',
|
|
57
|
+
defaultSshPort: 22,
|
|
58
|
+
defaultSshKeyPath: './engine-private/deploy/id_rsa',
|
|
59
|
+
remoteEnginePath: '/home/dd/engine',
|
|
60
|
+
// Latched in the root env, which the CronJob mounts from the host, so the
|
|
61
|
+
// decision survives the container that made it.
|
|
62
|
+
latchKey: 'VULTR_EGRESS_BLOCKED_AT',
|
|
63
|
+
env: {
|
|
64
|
+
apiKey: 'VULTR_API_KEY',
|
|
65
|
+
instanceId: 'VULTR_INSTANCE_ID',
|
|
66
|
+
threshold: 'VULTR_BANDWIDTH_THRESHOLD',
|
|
67
|
+
host: ['VULTR_VPS_IP', 'DEFAULT_SSH_HOST'],
|
|
68
|
+
user: ['VULTR_SSH_USER', 'DEFAULT_SSH_USER'],
|
|
69
|
+
keyPath: ['VULTR_SSH_KEY_PATH', 'DEFAULT_SSH_KEY_PATH'],
|
|
70
|
+
port: ['VULTR_SSH_PORT', 'DEFAULT_SSH_PORT'],
|
|
71
|
+
// The edge hub's forward proxy, named once in its canonical module so the two
|
|
72
|
+
// ends of it cannot disagree about the variable that configures it.
|
|
73
|
+
forwardProxyApiKey: FORWARD_PROXY.env.apiKey,
|
|
74
|
+
forwardProxyHost: FORWARD_PROXY.env.host,
|
|
75
|
+
forwardProxyPort: FORWARD_PROXY.env.port,
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* @method envFactory
|
|
81
|
+
* @description First non-empty value among a list of keys.
|
|
82
|
+
*
|
|
83
|
+
* Resolution is {@link module:src/server/environment.js.environmentValueFactory}'s — the process
|
|
84
|
+
* environment, then the deploy env `underpost env <deploy-id> <environment>`
|
|
85
|
+
* selects into `./.env`, then the underpost root env — rather than a second
|
|
86
|
+
* implementation of it, because the three callers differ: a CronJob container has
|
|
87
|
+
* its deploy env loaded into `process.env` by `loadCronDeployEnv`, an operator
|
|
88
|
+
* preparing a manual run has `./.env`, and an operator who ran `underpost env
|
|
89
|
+
* set` has the root env. Never logged — one of the keys this resolves is an API
|
|
90
|
+
* key.
|
|
91
|
+
* @param {string|Array<string>} keys - Environment variable name, or names in precedence order.
|
|
92
|
+
* @returns {string} The resolved value, or an empty string.
|
|
93
|
+
* @memberof UnderpostVultr
|
|
94
|
+
*/
|
|
95
|
+
const envFactory = (keys) => {
|
|
96
|
+
for (const key of Array.isArray(keys) ? keys : [keys]) {
|
|
97
|
+
const value = environmentValueFactory(key);
|
|
98
|
+
if (value) return value;
|
|
99
|
+
}
|
|
100
|
+
return '';
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* @method thresholdFactory
|
|
105
|
+
* @description Normalizes the configured trigger fraction.
|
|
106
|
+
*
|
|
107
|
+
* `80` and `0.80` are the same intent expressed two ways, and the first one
|
|
108
|
+
* silently never fires — a guard that never fires is worse than no guard, since
|
|
109
|
+
* it reads as protection. Both are accepted and resolve to the same ratio.
|
|
110
|
+
* @param {string|number} [raw] - Configured value.
|
|
111
|
+
* @param {number} [fallback] - Ratio used when nothing usable is configured.
|
|
112
|
+
* @returns {number} Ratio in `(0, 1]`.
|
|
113
|
+
* @memberof UnderpostVultr
|
|
114
|
+
*/
|
|
115
|
+
const thresholdFactory = (raw, fallback = UNDERPOST_VULTR.defaultThreshold) => {
|
|
116
|
+
const parsed = parseFloat(`${raw ?? ''}`.trim());
|
|
117
|
+
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
|
|
118
|
+
const ratio = parsed > 1 ? parsed / 100 : parsed;
|
|
119
|
+
return ratio > 1 ? 1 : ratio;
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* @method billingMonthFactory
|
|
124
|
+
* @description The `YYYY-MM` prefix the daily buckets of the current cycle carry.
|
|
125
|
+
*
|
|
126
|
+
* UTC, because the endpoint's day boundaries are UTC. A host in a negative
|
|
127
|
+
* offset would otherwise drop the current day for part of its evening.
|
|
128
|
+
* @param {Date} [now] - Reference instant.
|
|
129
|
+
* @returns {string} `YYYY-MM`.
|
|
130
|
+
* @memberof UnderpostVultr
|
|
131
|
+
*/
|
|
132
|
+
const billingMonthFactory = (now = new Date()) =>
|
|
133
|
+
`${now.getUTCFullYear()}-${`${now.getUTCMonth() + 1}`.padStart(2, '0')}`;
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* @method bandwidthTotalsFactory
|
|
137
|
+
* @description Folds the endpoint's daily buckets into one consumption figure.
|
|
138
|
+
*
|
|
139
|
+
* Scoped to the current month by default rather than summing every key
|
|
140
|
+
* returned. The response is a rolling window that can still carry the tail of
|
|
141
|
+
* the previous cycle, and those bytes are against a quota that has already
|
|
142
|
+
* reset — counting them reports a host as over budget on the first days of a
|
|
143
|
+
* month when it has barely spent anything.
|
|
144
|
+
*
|
|
145
|
+
* Both directions are returned separately because which of them is billable is
|
|
146
|
+
* a property of the account's plan, not of this code: `total` is the
|
|
147
|
+
* conservative reading and trips first, `outgoing` is the one that maps to
|
|
148
|
+
* egress alone.
|
|
149
|
+
* @param {object} [bandwidth] - `bandwidth` object from the Vultr response.
|
|
150
|
+
* @param {string} [month] - `YYYY-MM` to scope to; empty sums every bucket.
|
|
151
|
+
* @returns {{totalBytes: number, incomingBytes: number, outgoingBytes: number, days: number, dates: Array<string>}} Consumption for the window.
|
|
152
|
+
* @memberof UnderpostVultr
|
|
153
|
+
*/
|
|
154
|
+
const bandwidthTotalsFactory = ({ bandwidth = {}, month = billingMonthFactory() } = {}) => {
|
|
155
|
+
const dates = Object.keys(bandwidth || {})
|
|
156
|
+
.filter((date) => !month || `${date}`.startsWith(month))
|
|
157
|
+
.sort();
|
|
158
|
+
let incomingBytes = 0;
|
|
159
|
+
let outgoingBytes = 0;
|
|
160
|
+
for (const date of dates) {
|
|
161
|
+
const bucket = bandwidth[date] || {};
|
|
162
|
+
incomingBytes += Number(bucket.incoming_bytes) || 0;
|
|
163
|
+
outgoingBytes += Number(bucket.outgoing_bytes) || 0;
|
|
164
|
+
}
|
|
165
|
+
return { totalBytes: incomingBytes + outgoingBytes, incomingBytes, outgoingBytes, days: dates.length, dates };
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* @method quotaStateFactory
|
|
170
|
+
* @description The consumption decision: quota, effective trigger, and whether
|
|
171
|
+
* it has been crossed.
|
|
172
|
+
*
|
|
173
|
+
* A plan with no quota (`bandwidth: 0`, which Vultr uses for unmetered plans)
|
|
174
|
+
* yields `exceeded: false` and is reported as such rather than dividing by zero
|
|
175
|
+
* and blocking a host that cannot run out.
|
|
176
|
+
* @param {number} consumedBytes - Bytes counted for the cycle.
|
|
177
|
+
* @param {number} planBandwidthGB - Plan quota in GB.
|
|
178
|
+
* @param {number} [threshold] - Trigger fraction of the quota.
|
|
179
|
+
* @returns {{maxBytes: number, limitBytes: number, consumedBytes: number, ratio: number, exceeded: boolean, metered: boolean}} Decision inputs and outcome.
|
|
180
|
+
* @memberof UnderpostVultr
|
|
181
|
+
*/
|
|
182
|
+
const quotaStateFactory = ({
|
|
183
|
+
consumedBytes = 0,
|
|
184
|
+
planBandwidthGB = 0,
|
|
185
|
+
threshold = UNDERPOST_VULTR.defaultThreshold,
|
|
186
|
+
}) => {
|
|
187
|
+
const maxBytes = Math.max(0, Number(planBandwidthGB) || 0) * UNDERPOST_VULTR.bytesPerGB;
|
|
188
|
+
const limitBytes = maxBytes * threshold;
|
|
189
|
+
const metered = maxBytes > 0;
|
|
190
|
+
return {
|
|
191
|
+
maxBytes,
|
|
192
|
+
limitBytes,
|
|
193
|
+
consumedBytes,
|
|
194
|
+
ratio: metered ? consumedBytes / maxBytes : 0,
|
|
195
|
+
exceeded: metered && consumedBytes >= limitBytes,
|
|
196
|
+
metered,
|
|
197
|
+
};
|
|
198
|
+
};
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* @method formatBytes
|
|
202
|
+
* @description Byte count as GB, for log lines an operator has to read against
|
|
203
|
+
* a plan quota that is quoted in GB.
|
|
204
|
+
* @param {number} bytes - Byte count.
|
|
205
|
+
* @returns {string} e.g. `812.40 GB`.
|
|
206
|
+
* @memberof UnderpostVultr
|
|
207
|
+
*/
|
|
208
|
+
const formatBytes = (bytes) => `${((Number(bytes) || 0) / UNDERPOST_VULTR.bytesPerGB).toFixed(2)} GB`;
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* @method vultrUrlFactory
|
|
212
|
+
* @description One absolute API URL, with its query already encoded.
|
|
213
|
+
*
|
|
214
|
+
* Built here rather than left to the transport because the two transports below
|
|
215
|
+
* take a query differently — axios takes `params`, the proxy client takes a URL —
|
|
216
|
+
* and a request that differs by transport is a bug waiting for the day the
|
|
217
|
+
* fallback is used.
|
|
218
|
+
* @param {string} path - Path below `/v2`.
|
|
219
|
+
* @param {object} [params] - Query parameters.
|
|
220
|
+
* @returns {string} Absolute URL.
|
|
221
|
+
* @memberof UnderpostVultr
|
|
222
|
+
*/
|
|
223
|
+
const vultrUrlFactory = ({ path, params = {} }) => {
|
|
224
|
+
const url = new URL(`${UNDERPOST_VULTR.apiBaseUrl}${path}`);
|
|
225
|
+
for (const [key, value] of Object.entries(params))
|
|
226
|
+
if (value !== undefined && value !== null && `${value}` !== '') url.searchParams.set(key, `${value}`);
|
|
227
|
+
return url.href;
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* @method vultrGet
|
|
232
|
+
* @description The transport: through the edge hub's forward proxy when one is
|
|
233
|
+
* configured, straight out otherwise.
|
|
234
|
+
*
|
|
235
|
+
* The proxy is the point of this indirection. Vultr's API sees the address the
|
|
236
|
+
* request came from, and this job usually runs in a CronJob inside a homelab
|
|
237
|
+
* cluster — so a direct call arrives from a residential ISP address, while a
|
|
238
|
+
* proxied one arrives from the very VPS the job is metering. An API key scoped to
|
|
239
|
+
* the edge's address only works over the proxy.
|
|
240
|
+
*
|
|
241
|
+
* Non-2xx answers are returned rather than thrown, so both transports report a
|
|
242
|
+
* status the same way; only a transport failure throws.
|
|
243
|
+
* @param {string} url - Absolute URL.
|
|
244
|
+
* @param {string} apiKey - Vultr API key.
|
|
245
|
+
* @param {object} [proxy] - Forward proxy endpoint; ignored when it carries no key.
|
|
246
|
+
* @returns {Promise<{status: number, data: object}>} Status and parsed body.
|
|
247
|
+
* @memberof UnderpostVultr
|
|
248
|
+
*/
|
|
249
|
+
const vultrGet = async ({ url, apiKey, proxy }) => {
|
|
250
|
+
const headers = { Authorization: `Bearer ${apiKey}`, Accept: 'application/json' };
|
|
251
|
+
if (!proxy?.apiKey) {
|
|
252
|
+
const response = await axios.get(url, {
|
|
253
|
+
headers,
|
|
254
|
+
timeout: UNDERPOST_VULTR.requestTimeoutMs,
|
|
255
|
+
validateStatus: () => true,
|
|
256
|
+
});
|
|
257
|
+
return { status: response.status, data: response.data };
|
|
258
|
+
}
|
|
259
|
+
const { status, body } = await fetchViaForwardProxy(url, {
|
|
260
|
+
headers,
|
|
261
|
+
timeout: UNDERPOST_VULTR.requestTimeoutMs,
|
|
262
|
+
proxy,
|
|
263
|
+
});
|
|
264
|
+
try {
|
|
265
|
+
return { status, data: body ? JSON.parse(body) : {} };
|
|
266
|
+
} catch {
|
|
267
|
+
return { status, data: {} };
|
|
268
|
+
}
|
|
269
|
+
};
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* @method vultrRequest
|
|
273
|
+
* @description One authenticated Vultr API call.
|
|
274
|
+
*
|
|
275
|
+
* Errors are re-thrown with Vultr's own message and without the request config,
|
|
276
|
+
* because an axios error carries the `Authorization` header it was sent with and
|
|
277
|
+
* this module's failures are logged.
|
|
278
|
+
* @param {string} apiKey - Vultr API key.
|
|
279
|
+
* @param {string} path - Path below `/v2`.
|
|
280
|
+
* @param {object} [params] - Query parameters.
|
|
281
|
+
* @param {object} [proxy] - Forward proxy endpoint from {@link UnderpostVultr.resolveConfig}.
|
|
282
|
+
* @returns {Promise<object>} Response body.
|
|
283
|
+
* @throws {Error} With the API's status and message, never the credentials.
|
|
284
|
+
* @memberof UnderpostVultr
|
|
285
|
+
*/
|
|
286
|
+
const vultrRequest = async ({ apiKey, path, params = {}, proxy = null }) => {
|
|
287
|
+
let status = 0;
|
|
288
|
+
let data = {};
|
|
289
|
+
try {
|
|
290
|
+
({ status, data } = await vultrGet({ url: vultrUrlFactory({ path, params }), apiKey, proxy }));
|
|
291
|
+
} catch (error) {
|
|
292
|
+
const responseStatus = error?.response?.status;
|
|
293
|
+
const detail = error?.response?.data?.error || error?.message || 'request failed';
|
|
294
|
+
const code = error?.code || error?.cause?.code;
|
|
295
|
+
const proxyHint =
|
|
296
|
+
proxy?.apiKey && ['EHOSTUNREACH', 'ENETUNREACH', 'ECONNREFUSED', 'ETIMEDOUT'].includes(code)
|
|
297
|
+
? '; verify wg0 on the spoke; if only pods fail, re-run --wireguard-setup --client and restart wg0; if the host also fails, re-run --forward-proxy-server on the hub'
|
|
298
|
+
: '';
|
|
299
|
+
throw new Error(
|
|
300
|
+
`[vultr] GET /v2${path} failed${responseStatus ? ` (${responseStatus})` : ''}: ${detail}${proxyHint}`,
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
if (status < 200 || status >= 300)
|
|
304
|
+
throw new Error(`[vultr] GET /v2${path} failed (${status}): ${data?.error || 'request failed'}`);
|
|
305
|
+
return data;
|
|
306
|
+
};
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* @method planBandwidthGBFactory
|
|
310
|
+
* @description The monthly transfer quota, in GB, of the plan an instance runs.
|
|
311
|
+
*
|
|
312
|
+
* `/v2/plans` is paginated and the catalogue is longer than one page, so the
|
|
313
|
+
* cursor is followed rather than the first page searched — a plan that happens
|
|
314
|
+
* to sort past the page boundary would otherwise read as "not found" and, on a
|
|
315
|
+
* naive implementation, as a quota of zero.
|
|
316
|
+
* @param {string} apiKey - Vultr API key.
|
|
317
|
+
* @param {string} planId - Plan id from the instance record.
|
|
318
|
+
* @param {object} [proxy] - Forward proxy endpoint.
|
|
319
|
+
* @returns {Promise<number>} Quota in GB.
|
|
320
|
+
* @throws {Error} When the plan is absent from the catalogue.
|
|
321
|
+
* @memberof UnderpostVultr
|
|
322
|
+
*/
|
|
323
|
+
const planBandwidthGBFactory = async ({ apiKey, planId, proxy = null }) => {
|
|
324
|
+
let cursor = '';
|
|
325
|
+
for (let page = 0; page < UNDERPOST_VULTR.maxPlanPages; page++) {
|
|
326
|
+
const data = await vultrRequest({
|
|
327
|
+
apiKey,
|
|
328
|
+
path: '/plans',
|
|
329
|
+
params: { type: 'all', per_page: UNDERPOST_VULTR.plansPerPage, ...(cursor ? { cursor } : {}) },
|
|
330
|
+
proxy,
|
|
331
|
+
});
|
|
332
|
+
const match = (data?.plans || []).find((plan) => plan?.id === planId);
|
|
333
|
+
if (match) return Number(match.bandwidth) || 0;
|
|
334
|
+
cursor = `${data?.meta?.links?.next || ''}`.trim();
|
|
335
|
+
if (!cursor) break;
|
|
336
|
+
}
|
|
337
|
+
throw new Error(`[vultr] Plan ${planId} was not found in the plan catalogue; cannot resolve its bandwidth quota`);
|
|
338
|
+
};
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* @class UnderpostVultr
|
|
342
|
+
* @description Bandwidth accounting for the edge VPS, and the egress cut-off it
|
|
343
|
+
* triggers.
|
|
344
|
+
* @memberof UnderpostVultr
|
|
345
|
+
*/
|
|
346
|
+
class UnderpostVultr {
|
|
347
|
+
static API = {
|
|
348
|
+
/**
|
|
349
|
+
* @method callback
|
|
350
|
+
* @description CLI and cron entry point.
|
|
351
|
+
*
|
|
352
|
+
* Takes the same `(deployList, options)` shape every other
|
|
353
|
+
* {@link UnderpostCron} job does, so `underpost cron default vultr` dispatches
|
|
354
|
+
* to it unchanged. The deploy list is not used to select an instance — the
|
|
355
|
+
* edge hub is one machine for the whole cluster, exactly as its WireGuard
|
|
356
|
+
* peer registry is — but it is logged so a run is attributable.
|
|
357
|
+
* @param {string} [deployList] - Comma-separated deploy ids, from the cron dispatcher.
|
|
358
|
+
* @param {object} [options] - CLI flags.
|
|
359
|
+
* @returns {Promise<object>} Result from {@link UnderpostVultr.checkBandwidth}.
|
|
360
|
+
* @memberof UnderpostVultr
|
|
361
|
+
*/
|
|
362
|
+
callback: async function (deployList = 'default', options = {}) {
|
|
363
|
+
return await UnderpostVultr.API.checkBandwidth({ ...options, deployList });
|
|
364
|
+
},
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* @method resolveConfig
|
|
368
|
+
* @description Every input the guard needs, resolved once.
|
|
369
|
+
*
|
|
370
|
+
* CLI flags win over the environment so a one-off run can target another
|
|
371
|
+
* instance without editing any `.env`; the environment is the standing
|
|
372
|
+
* configuration. Single source of truth for the resolution, so no other
|
|
373
|
+
* method re-reads `process.env`.
|
|
374
|
+
* @param {object} [options] - CLI flags.
|
|
375
|
+
* @returns {object} Resolved configuration; `apiKey` is present but must never be logged.
|
|
376
|
+
* @memberof UnderpostVultr
|
|
377
|
+
*/
|
|
378
|
+
resolveConfig(options = {}) {
|
|
379
|
+
return {
|
|
380
|
+
apiKey: `${options.apiKey || ''}`.trim() || envFactory(UNDERPOST_VULTR.env.apiKey),
|
|
381
|
+
instanceId: `${options.instanceId || ''}`.trim() || envFactory(UNDERPOST_VULTR.env.instanceId),
|
|
382
|
+
threshold: thresholdFactory(options.threshold || envFactory(UNDERPOST_VULTR.env.threshold)),
|
|
383
|
+
host: `${options.host || ''}`.trim() || envFactory(UNDERPOST_VULTR.env.host),
|
|
384
|
+
user: `${options.user || ''}`.trim() || envFactory(UNDERPOST_VULTR.env.user) || UNDERPOST_VULTR.defaultSshUser,
|
|
385
|
+
keyPath:
|
|
386
|
+
`${options.keyPath || ''}`.trim() ||
|
|
387
|
+
envFactory(UNDERPOST_VULTR.env.keyPath) ||
|
|
388
|
+
UNDERPOST_VULTR.defaultSshKeyPath,
|
|
389
|
+
port: Number(options.port || envFactory(UNDERPOST_VULTR.env.port)) || UNDERPOST_VULTR.defaultSshPort,
|
|
390
|
+
// `total` is the conservative reading and trips first; `outgoing` counts
|
|
391
|
+
// egress alone, which is what a plan billing outbound-only meters.
|
|
392
|
+
metric: `${options.metric || 'total'}`.trim() === 'outgoing' ? 'outgoing' : 'total',
|
|
393
|
+
month: options.allDates === true ? '' : `${options.month || ''}`.trim() || billingMonthFactory(),
|
|
394
|
+
dryRun: options.dryRun === true,
|
|
395
|
+
force: options.force === true,
|
|
396
|
+
autoUnblock: options.autoUnblock === true,
|
|
397
|
+
// Resolved through the same env precedence as everything else, so the
|
|
398
|
+
// cron's deploy env can enable the proxy without a flag. An unset key
|
|
399
|
+
// means no proxy, and the API is called directly.
|
|
400
|
+
forwardProxy: {
|
|
401
|
+
apiKey: envFactory(UNDERPOST_VULTR.env.forwardProxyApiKey),
|
|
402
|
+
host: envFactory(UNDERPOST_VULTR.env.forwardProxyHost),
|
|
403
|
+
port: envFactory(UNDERPOST_VULTR.env.forwardProxyPort),
|
|
404
|
+
},
|
|
405
|
+
};
|
|
406
|
+
},
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* @method checkBandwidth
|
|
410
|
+
* @description Reads the instance's consumption for the cycle and enforces
|
|
411
|
+
* the threshold.
|
|
412
|
+
*
|
|
413
|
+
* Three calls, in the only order that works: the instance record names its
|
|
414
|
+
* plan, the plan carries the quota, and the bandwidth endpoint carries the
|
|
415
|
+
* consumption. Nothing is enforced until all three have answered — a guard
|
|
416
|
+
* that blocked an edge on a failed API call would be an outage caused by the
|
|
417
|
+
* thing meant to prevent one.
|
|
418
|
+
* @param {object} [options] - CLI flags; see {@link UnderpostVultr.resolveConfig}.
|
|
419
|
+
* @returns {Promise<{instanceId: string, plan: string, metric: string, month: string, consumedBytes: number, limitBytes: number, maxBytes: number, ratio: number, exceeded: boolean, enforced: boolean, latched: boolean}>} What was measured and what was done about it.
|
|
420
|
+
* @throws {Error} When credentials are missing or the API cannot be read.
|
|
421
|
+
* @memberof UnderpostVultr
|
|
422
|
+
*/
|
|
423
|
+
checkBandwidth: async function (options = {}) {
|
|
424
|
+
const config = UnderpostVultr.API.resolveConfig(options);
|
|
425
|
+
if (!config.apiKey) throw new Error(`[vultr] ${UNDERPOST_VULTR.env.apiKey} is not set`);
|
|
426
|
+
if (!config.instanceId) throw new Error(`[vultr] ${UNDERPOST_VULTR.env.instanceId} is not set`);
|
|
427
|
+
|
|
428
|
+
const proxy = config.forwardProxy;
|
|
429
|
+
const instance = (await vultrRequest({ apiKey: config.apiKey, path: `/instances/${config.instanceId}`, proxy }))
|
|
430
|
+
?.instance;
|
|
431
|
+
const planId = `${instance?.plan || ''}`.trim();
|
|
432
|
+
if (!planId) throw new Error(`[vultr] Instance ${config.instanceId} returned no plan id`);
|
|
433
|
+
|
|
434
|
+
const planBandwidthGB = await planBandwidthGBFactory({ apiKey: config.apiKey, planId, proxy });
|
|
435
|
+
const { bandwidth } = await vultrRequest({
|
|
436
|
+
apiKey: config.apiKey,
|
|
437
|
+
path: `/instances/${config.instanceId}/bandwidth`,
|
|
438
|
+
proxy,
|
|
439
|
+
});
|
|
440
|
+
|
|
441
|
+
const totals = bandwidthTotalsFactory({ bandwidth, month: config.month });
|
|
442
|
+
const consumedBytes = config.metric === 'outgoing' ? totals.outgoingBytes : totals.totalBytes;
|
|
443
|
+
const state = quotaStateFactory({ consumedBytes, planBandwidthGB, threshold: config.threshold });
|
|
444
|
+
const latchedAt = `${Underpost.env.get(UNDERPOST_VULTR.latchKey, undefined, { disableLog: true }) ?? ''}`.trim();
|
|
445
|
+
|
|
446
|
+
const summary = {
|
|
447
|
+
instanceId: config.instanceId,
|
|
448
|
+
plan: planId,
|
|
449
|
+
metric: config.metric,
|
|
450
|
+
month: config.month || 'all-dates',
|
|
451
|
+
days: totals.days,
|
|
452
|
+
consumed: formatBytes(consumedBytes),
|
|
453
|
+
incoming: formatBytes(totals.incomingBytes),
|
|
454
|
+
outgoing: formatBytes(totals.outgoingBytes),
|
|
455
|
+
triggerLimit: formatBytes(state.limitBytes),
|
|
456
|
+
planQuota: formatBytes(state.maxBytes),
|
|
457
|
+
usedPercent: `${(state.ratio * 100).toFixed(1)}%`,
|
|
458
|
+
threshold: `${(config.threshold * 100).toFixed(0)}%`,
|
|
459
|
+
// Which address Vultr saw the reads come from, which is the difference
|
|
460
|
+
// between a key scoped to the edge VPS working and failing.
|
|
461
|
+
via: proxy.apiKey ? `forward-proxy ${proxy.host || FORWARD_PROXY.defaultHost}` : 'direct',
|
|
462
|
+
};
|
|
463
|
+
|
|
464
|
+
if (!state.metered) {
|
|
465
|
+
logger.info('Vultr plan reports no metered bandwidth quota; nothing to enforce', summary);
|
|
466
|
+
return { ...state, ...summary, enforced: false, latched: !!latchedAt };
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
if (!state.exceeded) {
|
|
470
|
+
logger.info('Vultr bandwidth within budget', summary);
|
|
471
|
+
// A latch that outlives the cycle it was set in would suppress the next
|
|
472
|
+
// real trigger, so it is cleared as soon as usage is back under the
|
|
473
|
+
// threshold — but the host stays blocked until someone says otherwise.
|
|
474
|
+
if (latchedAt) {
|
|
475
|
+
if (!config.dryRun) Underpost.env.delete(UNDERPOST_VULTR.latchKey);
|
|
476
|
+
if (config.autoUnblock) await UnderpostVultr.API.setEdgeEgress({ config, blocked: false });
|
|
477
|
+
else
|
|
478
|
+
logger.warn('Edge egress is still blocked from a previous cycle; unblock it when you are ready', {
|
|
479
|
+
host: config.host,
|
|
480
|
+
blockedAt: latchedAt,
|
|
481
|
+
next: `underpost ip --unblock-all-egress (on ${config.host || 'the edge VPS'})`,
|
|
482
|
+
});
|
|
483
|
+
}
|
|
484
|
+
return { ...state, ...summary, enforced: false, latched: false };
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
logger.warn('Vultr bandwidth threshold reached; edge egress will be blocked', summary);
|
|
488
|
+
|
|
489
|
+
if (latchedAt && !config.force) {
|
|
490
|
+
logger.info('Edge egress was already blocked for this cycle; not re-applying', {
|
|
491
|
+
host: config.host,
|
|
492
|
+
blockedAt: latchedAt,
|
|
493
|
+
next: 'pass --force to re-apply',
|
|
494
|
+
});
|
|
495
|
+
return { ...state, ...summary, enforced: false, latched: true };
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
const enforced = await UnderpostVultr.API.setEdgeEgress({ config, blocked: true });
|
|
499
|
+
if (enforced && !config.dryRun) Underpost.env.set(UNDERPOST_VULTR.latchKey, new Date().toISOString());
|
|
500
|
+
return { ...state, ...summary, enforced, latched: enforced };
|
|
501
|
+
},
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* @method setEdgeEgress
|
|
505
|
+
* @description Blocks or restores outbound traffic on the edge VPS over SSH.
|
|
506
|
+
*
|
|
507
|
+
* The command is run on the VPS rather than here because the nftables rules
|
|
508
|
+
* belong to that host — this process usually runs in a CronJob container
|
|
509
|
+
* inside a spoke cluster, on the far side of the tunnel the rules govern.
|
|
510
|
+
*
|
|
511
|
+
* `underpost ip` is preferred when the CLI is installed globally and the
|
|
512
|
+
* checked-out engine is the fallback, so a VPS provisioned either way is
|
|
513
|
+
* reachable. The command already elevates internally, so it is not wrapped
|
|
514
|
+
* in `sudo` here.
|
|
515
|
+
* @param {object} config - Resolved configuration from {@link UnderpostVultr.resolveConfig}.
|
|
516
|
+
* @param {boolean} blocked - True to block egress, false to restore it.
|
|
517
|
+
* @returns {Promise<boolean>} True when the remote command succeeded.
|
|
518
|
+
* @memberof UnderpostVultr
|
|
519
|
+
*/
|
|
520
|
+
setEdgeEgress: async function ({ config, blocked }) {
|
|
521
|
+
const flag = blocked ? '--block-all-egress' : '--unblock-all-egress';
|
|
522
|
+
if (!config.host) {
|
|
523
|
+
logger.error('No edge host configured; cannot reach the VPS to change its egress', {
|
|
524
|
+
set: UNDERPOST_VULTR.env.host.join(' or '),
|
|
525
|
+
});
|
|
526
|
+
return false;
|
|
527
|
+
}
|
|
528
|
+
const command = [
|
|
529
|
+
'set -e',
|
|
530
|
+
'if command -v underpost >/dev/null 2>&1; then',
|
|
531
|
+
` underpost ip ${flag}`,
|
|
532
|
+
'else',
|
|
533
|
+
` cd ${UNDERPOST_VULTR.remoteEnginePath} && node bin ip ${flag}`,
|
|
534
|
+
'fi',
|
|
535
|
+
].join('\n');
|
|
536
|
+
|
|
537
|
+
if (config.dryRun) {
|
|
538
|
+
logger.info('[dry-run] would run on the edge VPS', {
|
|
539
|
+
target: `${config.user}@${config.host}:${config.port}`,
|
|
540
|
+
command: `underpost ip ${flag}`,
|
|
541
|
+
});
|
|
542
|
+
return false;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
const result = await Underpost.ssh.sshExecBatch({
|
|
546
|
+
host: config.host,
|
|
547
|
+
port: config.port,
|
|
548
|
+
user: config.user,
|
|
549
|
+
keyPath: config.keyPath,
|
|
550
|
+
command,
|
|
551
|
+
});
|
|
552
|
+
if (!result.ok) {
|
|
553
|
+
logger.error(`Failed to run 'underpost ip ${flag}' on the edge VPS`, {
|
|
554
|
+
target: `${config.user}@${config.host}:${config.port}`,
|
|
555
|
+
code: result.code,
|
|
556
|
+
stderr: `${result.stderr || ''}`.slice(-400),
|
|
557
|
+
});
|
|
558
|
+
return false;
|
|
559
|
+
}
|
|
560
|
+
if (blocked)
|
|
561
|
+
logger.warn('Edge egress blocked; every hostname behind the hub is now offline', {
|
|
562
|
+
host: config.host,
|
|
563
|
+
restore: `underpost ip --unblock-all-egress (on ${config.host})`,
|
|
564
|
+
});
|
|
565
|
+
else logger.info('Edge egress restored', { host: config.host });
|
|
566
|
+
return true;
|
|
567
|
+
},
|
|
568
|
+
};
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
export default UnderpostVultr;
|
|
572
|
+
|
|
573
|
+
export {
|
|
574
|
+
UNDERPOST_VULTR,
|
|
575
|
+
bandwidthTotalsFactory,
|
|
576
|
+
billingMonthFactory,
|
|
577
|
+
envFactory,
|
|
578
|
+
formatBytes,
|
|
579
|
+
planBandwidthGBFactory,
|
|
580
|
+
quotaStateFactory,
|
|
581
|
+
thresholdFactory,
|
|
582
|
+
vultrUrlFactory,
|
|
583
|
+
};
|