sublera-cli 0.0.0-stage → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +172 -0
- package/LICENSE +21 -0
- package/README.md +585 -2
- package/SECURITY.md +78 -0
- package/THIRD_PARTY_NOTICES.md +37 -0
- package/dist/agent.d.ts +270 -0
- package/dist/agent.js +1867 -0
- package/dist/commands.d.ts +76 -0
- package/dist/commands.js +841 -0
- package/dist/config.d.ts +27 -0
- package/dist/config.js +95 -0
- package/dist/engine/binaryManager.d.ts +67 -0
- package/dist/engine/binaryManager.js +590 -0
- package/dist/engine/gguf.d.ts +88 -0
- package/dist/engine/gguf.js +458 -0
- package/dist/engine/llamaRunner.d.ts +78 -0
- package/dist/engine/llamaRunner.js +282 -0
- package/dist/engine/modelManager.d.ts +63 -0
- package/dist/engine/modelManager.js +510 -0
- package/dist/engine/selfCheck.d.ts +34 -0
- package/dist/engine/selfCheck.js +108 -0
- package/dist/exitCodes.d.ts +20 -0
- package/dist/exitCodes.js +20 -0
- package/dist/hardware/gpuMonitor.d.ts +32 -0
- package/dist/hardware/gpuMonitor.js +378 -0
- package/dist/hardware/sleepGuard.d.ts +42 -0
- package/dist/hardware/sleepGuard.js +149 -0
- package/dist/hardware/types.d.ts +40 -0
- package/dist/hardware/types.js +2 -0
- package/dist/health.d.ts +17 -0
- package/dist/health.js +38 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.js +268 -0
- package/dist/logger.d.ts +53 -0
- package/dist/logger.js +201 -0
- package/dist/network/proxy.d.ts +58 -0
- package/dist/network/proxy.js +328 -0
- package/dist/network/streamProxy.d.ts +51 -0
- package/dist/network/streamProxy.js +286 -0
- package/dist/network/tunnelClient.d.ts +54 -0
- package/dist/network/tunnelClient.js +257 -0
- package/dist/settings.d.ts +53 -0
- package/dist/settings.js +171 -0
- package/dist/state.d.ts +52 -0
- package/dist/state.js +71 -0
- package/dist/storageCommand.d.ts +22 -0
- package/dist/storageCommand.js +81 -0
- package/dist/types.d.ts +222 -0
- package/dist/types.js +43 -0
- package/dist/ui/progress.d.ts +29 -0
- package/dist/ui/progress.js +82 -0
- package/dist/ui/terminalDashboard.d.ts +88 -0
- package/dist/ui/terminalDashboard.js +295 -0
- package/dist/util/env.d.ts +3 -0
- package/dist/util/env.js +6 -0
- package/dist/util/exec.d.ts +23 -0
- package/dist/util/exec.js +116 -0
- package/dist/util/format.d.ts +47 -0
- package/dist/util/format.js +125 -0
- package/dist/util/fs.d.ts +18 -0
- package/dist/util/fs.js +140 -0
- package/dist/util/http.d.ts +28 -0
- package/dist/util/http.js +128 -0
- package/dist/util/instanceLock.d.ts +3 -0
- package/dist/util/instanceLock.js +108 -0
- package/dist/util/open.d.ts +16 -0
- package/dist/util/open.js +65 -0
- package/dist/util/storage.d.ts +10 -0
- package/dist/util/storage.js +69 -0
- package/dist/version.d.ts +15 -0
- package/dist/version.js +29 -0
- package/package.json +68 -4
package/dist/agent.js
ADDED
|
@@ -0,0 +1,1867 @@
|
|
|
1
|
+
import { EventEmitter } from 'node:events';
|
|
2
|
+
import { basename } from 'node:path';
|
|
3
|
+
import { apiUrl, hostEndpoint } from './config.js';
|
|
4
|
+
import { GpuMonitor, applyGpuOverride, checkFit } from './hardware/gpuMonitor.js';
|
|
5
|
+
import { SleepGuard } from './hardware/sleepGuard.js';
|
|
6
|
+
import { ensureLlamaServer } from './engine/binaryManager.js';
|
|
7
|
+
import { downloadModel, listCachedModels, verifyModelFile } from './engine/modelManager.js';
|
|
8
|
+
import { checkModelUrl, estimateVramMb, inspectHuggingFace, parseHfUrl, paramBFromName, quantFromName } from './engine/gguf.js';
|
|
9
|
+
import { LlamaRunner } from './engine/llamaRunner.js';
|
|
10
|
+
import { compareFingerprints, probeEngine } from './engine/selfCheck.js';
|
|
11
|
+
import { simulateChat, streamChat } from './network/streamProxy.js';
|
|
12
|
+
import { TunnelClient } from './network/tunnelClient.js';
|
|
13
|
+
import { TerminalDashboard, initialState } from './ui/terminalDashboard.js';
|
|
14
|
+
import { activeRequestsOf, finiteNumber, isRequestId, validTime } from './types.js';
|
|
15
|
+
import { fetchJson, HttpError } from './util/http.js';
|
|
16
|
+
import { execFile, sleep } from './util/exec.js';
|
|
17
|
+
import { clearAgentState, readAgentState, saveAgentState } from './state.js';
|
|
18
|
+
import { VERSION } from './version.js';
|
|
19
|
+
import { openUrl, printable } from './util/open.js';
|
|
20
|
+
import { acquireInstanceLock } from './util/instanceLock.js';
|
|
21
|
+
import { atomicWrite } from './util/fs.js';
|
|
22
|
+
import { healthPath } from './health.js';
|
|
23
|
+
// setTimeout fires at once for delays above 2^31-1 ms; long bookings re-arm instead.
|
|
24
|
+
const MAX_TIMER_MS = 2_000_000_000;
|
|
25
|
+
/** How long an ended booking's id is remembered, so a stale status answer cannot bring it back. */
|
|
26
|
+
const ENDED_TTL_MS = 3_600_000;
|
|
27
|
+
/** Request statuses the server lists while a request is live on this host. */
|
|
28
|
+
const LIVE_STATUSES = new Set(['claimed', 'loading', 'verifying', 'active']);
|
|
29
|
+
/** Server frames that name a request; one with a malformed requestId is ignored. */
|
|
30
|
+
const REQUEST_FRAMES = new Set(['load', 'verified', 'booking', 'reject']);
|
|
31
|
+
/** An ISO time for logs, or null when `ms` is not a time `Date` can show (toISOString would throw). */
|
|
32
|
+
function isoTime(ms) {
|
|
33
|
+
if (ms == null || !Number.isFinite(ms))
|
|
34
|
+
return null;
|
|
35
|
+
const date = new Date(ms);
|
|
36
|
+
return Number.isNaN(date.getTime()) ? null : date.toISOString();
|
|
37
|
+
}
|
|
38
|
+
/** Aborts when any of the signals aborts (AbortSignal.any needs Node 20.3). */
|
|
39
|
+
function anySignal(signals) {
|
|
40
|
+
const controller = new AbortController();
|
|
41
|
+
for (const signal of signals) {
|
|
42
|
+
if (signal.aborted) {
|
|
43
|
+
controller.abort(signal.reason);
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
signal.addEventListener('abort', () => controller.abort(signal.reason), { once: true });
|
|
47
|
+
}
|
|
48
|
+
return controller.signal;
|
|
49
|
+
}
|
|
50
|
+
function simulatedGpu(override) {
|
|
51
|
+
const empty = { gpus: [], primary: null, source: 'simulation', platform: process.platform, checkedAt: Date.now(), warnings: [] };
|
|
52
|
+
if (!override)
|
|
53
|
+
return { ...empty, warnings: ['Simulation mode: no GPU was scanned.'] };
|
|
54
|
+
return applyGpuOverride(empty, { name: override.name ?? 'Simulated GPU', vramMb: override.vramMb ?? 24_576 });
|
|
55
|
+
}
|
|
56
|
+
export class Agent extends EventEmitter {
|
|
57
|
+
options;
|
|
58
|
+
logger;
|
|
59
|
+
gpuMonitor;
|
|
60
|
+
sleepGuard;
|
|
61
|
+
dashboard;
|
|
62
|
+
tunnel = null;
|
|
63
|
+
runner = null;
|
|
64
|
+
install = null;
|
|
65
|
+
runnerModelPath = null;
|
|
66
|
+
runnerContext = null;
|
|
67
|
+
/** The --parallel count the running llama-server was started with. */
|
|
68
|
+
runnerSlotCount = null;
|
|
69
|
+
gpu;
|
|
70
|
+
model = null;
|
|
71
|
+
cachedModels = [];
|
|
72
|
+
me = null;
|
|
73
|
+
meTimer = null;
|
|
74
|
+
gpuTimer = null;
|
|
75
|
+
activeJobs = new Map();
|
|
76
|
+
stopped = false;
|
|
77
|
+
startup = new AbortController();
|
|
78
|
+
/** Aborts the download of a request the server stopped (released, cancelled, or taken back). */
|
|
79
|
+
loadAbort = null;
|
|
80
|
+
/** A model download or engine start that replaces the runner: new inference waits. */
|
|
81
|
+
loadingModel = false;
|
|
82
|
+
/** Any load frame in progress, including one that reuses the running model. */
|
|
83
|
+
loadInFlight = false;
|
|
84
|
+
/** llama-server slots the last load asked for: more than one only for a shared booking. */
|
|
85
|
+
bookingSlots = 1;
|
|
86
|
+
/** Every booked request, keyed by requestId. */
|
|
87
|
+
bookings = new Map();
|
|
88
|
+
/**
|
|
89
|
+
* Bookings that ended in this session (end timer, reject, or dropped by the status poll) and
|
|
90
|
+
* when. Their earned share is already settled, so they are never added again.
|
|
91
|
+
*/
|
|
92
|
+
endedIds = new Map();
|
|
93
|
+
/** Server clock minus this machine's clock, from the last `/api/agent/me` answer. */
|
|
94
|
+
serverOffset = 0;
|
|
95
|
+
/** Requests the last status poll listed as live: claimed, loading, verifying or active. */
|
|
96
|
+
liveRequestIds = new Set();
|
|
97
|
+
/** The id of `me.activeRequest` (the server's newest live request) in the last status answer. */
|
|
98
|
+
serverNewestId = null;
|
|
99
|
+
pending = null;
|
|
100
|
+
/** What the dashboard shows with no request: the public model, or the model loaded with --model. */
|
|
101
|
+
startupJob = null;
|
|
102
|
+
tokenEarnedNanos = 0;
|
|
103
|
+
/** Earned share of bookings that ended or were stopped in this session. */
|
|
104
|
+
settledEarnedNanos = 0;
|
|
105
|
+
settledGuaranteedNanos = 0;
|
|
106
|
+
meInitialWallet = null;
|
|
107
|
+
lastSpeed = null;
|
|
108
|
+
lastTtft = null;
|
|
109
|
+
persisted = null;
|
|
110
|
+
/** True while the restore after a restart works on state.json. */
|
|
111
|
+
restoring = false;
|
|
112
|
+
stateWrite = Promise.resolve();
|
|
113
|
+
online = false;
|
|
114
|
+
onlineWaiters = [];
|
|
115
|
+
keyHandler = null;
|
|
116
|
+
heartbeatAt = null;
|
|
117
|
+
bookingTimer = null;
|
|
118
|
+
logLines = [];
|
|
119
|
+
/** The engine's answer to the self-check prompt right after the current model was loaded. */
|
|
120
|
+
engineBaseline = null;
|
|
121
|
+
selfCheckAbort = null;
|
|
122
|
+
draining = false;
|
|
123
|
+
runStarted = false;
|
|
124
|
+
stopPromise = null;
|
|
125
|
+
messageTasks = new Set();
|
|
126
|
+
healthTimer = null;
|
|
127
|
+
healthWrite = Promise.resolve();
|
|
128
|
+
constructor(options) {
|
|
129
|
+
super();
|
|
130
|
+
this.options = options;
|
|
131
|
+
this.logger = options.logger;
|
|
132
|
+
this.gpuMonitor = new GpuMonitor();
|
|
133
|
+
this.sleepGuard = new SleepGuard({ logger: options.logger });
|
|
134
|
+
this.gpu = simulatedGpu(options.gpuOverride ?? null);
|
|
135
|
+
const state = initialState(options.config.server);
|
|
136
|
+
this.dashboard = new TerminalDashboard(state, {
|
|
137
|
+
enabled: !options.json,
|
|
138
|
+
color: Boolean(process.stdout.isTTY),
|
|
139
|
+
unicode: process.platform === 'win32' ? true : true,
|
|
140
|
+
});
|
|
141
|
+
if (options.publicMode)
|
|
142
|
+
this.dashboard.update({ publicMode: true, priceOutUsd: this.priceOutUsd(), priceHourUsd: this.priceHourUsd() });
|
|
143
|
+
}
|
|
144
|
+
get dashboardState() { return this.dashboard.current; }
|
|
145
|
+
/** Every booked request, soonest end first. */
|
|
146
|
+
get activeBookings() {
|
|
147
|
+
return this.sortedBookings().map((booking) => this.bookingView(booking));
|
|
148
|
+
}
|
|
149
|
+
/** The booking for this request, or null when it is not (or no longer) booked here. */
|
|
150
|
+
booking(requestId) {
|
|
151
|
+
const booking = this.bookings.get(requestId);
|
|
152
|
+
return booking ? this.bookingView(booking) : null;
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* True while something other than `requestId` still needs this agent: public selling, another
|
|
156
|
+
* booking, another request being loaded, or another request the server lists as live. A seat on
|
|
157
|
+
* the loaded model never gets a load frame, so during its model check only the server's list shows it.
|
|
158
|
+
*/
|
|
159
|
+
servingOthers(requestId) {
|
|
160
|
+
if (this.options.publicMode)
|
|
161
|
+
return true;
|
|
162
|
+
for (const id of this.bookings.keys())
|
|
163
|
+
if (id !== requestId)
|
|
164
|
+
return true;
|
|
165
|
+
for (const id of this.liveRequestIds)
|
|
166
|
+
if (id !== requestId && !this.endedIds.has(id))
|
|
167
|
+
return true;
|
|
168
|
+
return Boolean(this.pending && this.pending.requestId !== requestId && (this.pending.holdsWake || this.loadInFlight));
|
|
169
|
+
}
|
|
170
|
+
/** This machine's estimate of the server clock. */
|
|
171
|
+
serverTime() {
|
|
172
|
+
return Date.now() + this.serverOffset;
|
|
173
|
+
}
|
|
174
|
+
/** Remembers the server clock from a status answer, so ends are judged by server time even when this clock is off. */
|
|
175
|
+
noteServerClock(me) {
|
|
176
|
+
const serverNow = finiteNumber(me.serverNow);
|
|
177
|
+
if (serverNow != null && serverNow > 0)
|
|
178
|
+
this.serverOffset = serverNow - Date.now();
|
|
179
|
+
}
|
|
180
|
+
/** The loaded request that waits for its model check, or a loaded benchmark. Restarting llama-server would break it. */
|
|
181
|
+
checkingRequest() {
|
|
182
|
+
const pending = this.pending;
|
|
183
|
+
if (!pending?.loaded)
|
|
184
|
+
return null;
|
|
185
|
+
// A request the server stopped listing no longer counts; a benchmark is not listed and ends with a reject.
|
|
186
|
+
return pending.holdsWake || pending.requestId.startsWith('bench_') ? pending : null;
|
|
187
|
+
}
|
|
188
|
+
/** Keeps the machine awake for a booking (the same call for a booking frame and a status poll). */
|
|
189
|
+
holdWakeLock() {
|
|
190
|
+
void this.sleepGuard.acquire('dedicated booking window').then(() => {
|
|
191
|
+
if (!this.stopped)
|
|
192
|
+
this.dashboard.update({ wakeLock: this.sleepGuard.status().active });
|
|
193
|
+
}, () => undefined);
|
|
194
|
+
}
|
|
195
|
+
bookingView(booking) {
|
|
196
|
+
const { requestId, verified, timerStartedAt, timerEndsAt, hours, minTokS, guaranteedNanos, shared, tokS } = booking;
|
|
197
|
+
return { requestId, verified, timerStartedAt, timerEndsAt, hours, minTokS, guaranteedNanos, shared, tokS };
|
|
198
|
+
}
|
|
199
|
+
/** Bookings, a pending request, or a real load keep the machine awake. */
|
|
200
|
+
get bookingWake() {
|
|
201
|
+
return this.loadingModel || this.bookings.size > 0 || Boolean(this.pending?.holdsWake);
|
|
202
|
+
}
|
|
203
|
+
priceOutUsd() {
|
|
204
|
+
return this.options.priceOutUsd ?? this.options.price ?? 0.2;
|
|
205
|
+
}
|
|
206
|
+
priceHourUsd() {
|
|
207
|
+
return this.options.publicMode ? (this.options.priceHourUsd ?? 0.5) : 0;
|
|
208
|
+
}
|
|
209
|
+
get health() {
|
|
210
|
+
const connected = this.online && this.tunnel?.connected === true;
|
|
211
|
+
const engine = this.options.simulation ? 'simulation' : this.runner?.status || 'not-loaded';
|
|
212
|
+
return {
|
|
213
|
+
schemaVersion: 1, agentVersion: VERSION, pid: process.pid, updatedAt: Date.now(),
|
|
214
|
+
live: !this.stopped, ready: Boolean(connected && !this.stopped && !this.draining && !this.loadingModel
|
|
215
|
+
&& (this.options.simulation || this.runner?.status === 'ready')),
|
|
216
|
+
draining: this.draining, simulation: this.options.simulation,
|
|
217
|
+
tunnel: this.tunnel?.state || 'closed', engine, activeJobs: this.activeJobs.size, loading: this.loadingModel,
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
publishHealth() {
|
|
221
|
+
const snapshot = JSON.stringify(this.health);
|
|
222
|
+
this.healthWrite = this.healthWrite.then(() => atomicWrite(healthPath(), `${snapshot}\n`)).catch(() => {
|
|
223
|
+
this.logger.warn('Could not write local health snapshot; check disk space and permissions');
|
|
224
|
+
});
|
|
225
|
+
return this.healthWrite;
|
|
226
|
+
}
|
|
227
|
+
/** Reject new work, finish in-flight inference up to a bounded grace period,
|
|
228
|
+
* then cancel leftovers and preserve booking state for server reconciliation. */
|
|
229
|
+
async drain(timeoutMs = this.options.drainTimeoutMs ?? 30_000) {
|
|
230
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs < 0 || timeoutMs > 300_000)
|
|
231
|
+
throw new Error('Drain timeout must be between 0 and 300000 ms');
|
|
232
|
+
if (this.stopped)
|
|
233
|
+
return this.stopPromise ?? Promise.resolve();
|
|
234
|
+
this.draining = true;
|
|
235
|
+
this.selfCheckAbort?.abort();
|
|
236
|
+
this.loadAbort?.controller.abort(new Error('Agent draining'));
|
|
237
|
+
this.logger.info('Agent draining; new work will be refused', { timeoutMs, activeJobs: this.activeJobs.size });
|
|
238
|
+
await this.publishHealth();
|
|
239
|
+
const deadline = Date.now() + timeoutMs;
|
|
240
|
+
while (!this.stopped && this.activeJobs.size > 0 && Date.now() < deadline)
|
|
241
|
+
await sleep(25);
|
|
242
|
+
await this.stop(true);
|
|
243
|
+
}
|
|
244
|
+
waitForOnline(timeoutMs = 20_000) {
|
|
245
|
+
if (this.online && this.tunnel?.connected)
|
|
246
|
+
return Promise.resolve();
|
|
247
|
+
if (this.stopped || this.draining)
|
|
248
|
+
return Promise.reject(new Error('Agent is stopped or draining'));
|
|
249
|
+
return new Promise((resolve, reject) => {
|
|
250
|
+
const timer = setTimeout(() => {
|
|
251
|
+
this.onlineWaiters = this.onlineWaiters.filter((w) => w.timer !== timer);
|
|
252
|
+
reject(new Error('Timed out waiting for the host tunnel'));
|
|
253
|
+
}, timeoutMs);
|
|
254
|
+
this.onlineWaiters.push({ resolve, reject, timer });
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
markOnline() {
|
|
258
|
+
if (this.stopped || this.draining || !this.tunnel?.connected)
|
|
259
|
+
return;
|
|
260
|
+
this.online = true;
|
|
261
|
+
for (const waiter of this.onlineWaiters) {
|
|
262
|
+
clearTimeout(waiter.timer);
|
|
263
|
+
waiter.resolve();
|
|
264
|
+
}
|
|
265
|
+
this.onlineWaiters = [];
|
|
266
|
+
this.emit('online');
|
|
267
|
+
}
|
|
268
|
+
/** After a crash or restart, reload the booked model and resume every booking saved in state.json. */
|
|
269
|
+
async restorePersistedBooking() {
|
|
270
|
+
// Only run() calls this, while it holds the instance lock, so state.json is this agent's file.
|
|
271
|
+
this.restoring = true;
|
|
272
|
+
try {
|
|
273
|
+
return await this.restoreOwned();
|
|
274
|
+
}
|
|
275
|
+
finally {
|
|
276
|
+
this.restoring = false;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
async restoreOwned() {
|
|
280
|
+
// readAgentState checks every number and drops bookings without a valid end time.
|
|
281
|
+
const state = await readAgentState();
|
|
282
|
+
if (!state || this.stopped)
|
|
283
|
+
return false;
|
|
284
|
+
let saved = state.bookings.filter((booking) => booking.timerEndsAt != null && booking.timerEndsAt >= Date.now() - 60_000);
|
|
285
|
+
if (!saved.length) {
|
|
286
|
+
await clearAgentState();
|
|
287
|
+
return false;
|
|
288
|
+
}
|
|
289
|
+
// If the server is reachable, trust its view of the bookings before loading a
|
|
290
|
+
// large cached model. A cancelled/completed booking must not be resurrected.
|
|
291
|
+
let fetched = null;
|
|
292
|
+
try {
|
|
293
|
+
fetched = await fetchJson(apiUrl(this.options.config.server, '/api/agent/me'), {
|
|
294
|
+
token: this.options.config.token,
|
|
295
|
+
timeoutMs: 5_000,
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
catch {
|
|
299
|
+
// Server may be temporarily unreachable; the local snapshot is still useful.
|
|
300
|
+
}
|
|
301
|
+
if (this.stopped)
|
|
302
|
+
return false;
|
|
303
|
+
if (fetched && typeof fetched === 'object') {
|
|
304
|
+
const me = fetched;
|
|
305
|
+
this.noteServerClock(me);
|
|
306
|
+
this.serverNewestId = isRequestId(me.activeRequest?.id) ? me.activeRequest.id : null;
|
|
307
|
+
// activeRequestsOf checks the ids and numbers of the rows.
|
|
308
|
+
const live = new Map(activeRequestsOf(me).map((row) => [row.id, row]));
|
|
309
|
+
const now = this.serverTime();
|
|
310
|
+
saved = saved
|
|
311
|
+
.filter((booking) => live.has(booking.requestId))
|
|
312
|
+
.map((booking) => {
|
|
313
|
+
const row = live.get(booking.requestId);
|
|
314
|
+
return { ...booking, timerEndsAt: row.timerEndsAt || booking.timerEndsAt, timerStartedAt: row.timerStartedAt || booking.timerStartedAt };
|
|
315
|
+
})
|
|
316
|
+
.filter((booking) => booking.timerEndsAt != null && booking.timerEndsAt >= now);
|
|
317
|
+
if (!saved.length) {
|
|
318
|
+
await clearAgentState();
|
|
319
|
+
return false;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
const first = saved[0];
|
|
323
|
+
// Every booking runs on the one loaded model; a record for another file cannot be served.
|
|
324
|
+
const modelKey = (model) => model.path || model.file;
|
|
325
|
+
const restorable = saved.filter((booking) => modelKey(booking.model) === modelKey(first.model));
|
|
326
|
+
if (restorable.length < saved.length) {
|
|
327
|
+
this.logger.warn('Dropping saved bookings for a different model', { requestIds: saved.filter((b) => !restorable.includes(b)).map((b) => b.requestId).join(',') });
|
|
328
|
+
}
|
|
329
|
+
// Restart llama-server with the slots and context the bookings had before the restart.
|
|
330
|
+
this.bookingSlots = Math.min(16, Math.max(1, ...restorable.map((booking) => Math.round(Number(booking.slots) || 1))));
|
|
331
|
+
const context = Math.max(...restorable.map((booking) => Number(booking.context) || 0)) || this.options.context || 8192;
|
|
332
|
+
try {
|
|
333
|
+
let model;
|
|
334
|
+
if (this.options.simulation) {
|
|
335
|
+
model = {
|
|
336
|
+
path: first.model.path || null,
|
|
337
|
+
sourceUrl: first.model.url || first.model.path || '',
|
|
338
|
+
file: first.model.file,
|
|
339
|
+
modelId: first.model.modelId,
|
|
340
|
+
label: first.model.label,
|
|
341
|
+
repo: first.model.repo,
|
|
342
|
+
quant: first.model.quant,
|
|
343
|
+
paramB: first.model.paramB,
|
|
344
|
+
layers: first.model.layers,
|
|
345
|
+
contextTrain: first.model.contextTrain,
|
|
346
|
+
sha256: first.model.sha256,
|
|
347
|
+
verified: first.model.verified,
|
|
348
|
+
needsMb: first.model.needsMb,
|
|
349
|
+
size: first.model.size,
|
|
350
|
+
architecture: first.model.architecture,
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
else {
|
|
354
|
+
if (!first.model.path)
|
|
355
|
+
throw new Error('The cached model path is missing');
|
|
356
|
+
model = await this.loadModel(first.model.path, {
|
|
357
|
+
context,
|
|
358
|
+
requestId: first.requestId,
|
|
359
|
+
expectedSha: first.model.sha256,
|
|
360
|
+
expectedLayers: first.model.layers,
|
|
361
|
+
file: first.model.file,
|
|
362
|
+
});
|
|
363
|
+
// A stop during the load: run() frees the engine, and stop() decides about state.json.
|
|
364
|
+
if (this.stopped)
|
|
365
|
+
return false;
|
|
366
|
+
}
|
|
367
|
+
this.model = model;
|
|
368
|
+
await this.sleepGuard.acquire('resuming a booked model after a process restart');
|
|
369
|
+
// Bookings added after a stop would keep end timers that nothing clears.
|
|
370
|
+
if (this.stopped)
|
|
371
|
+
return false;
|
|
372
|
+
for (const booking of restorable) {
|
|
373
|
+
this.upsertBooking(booking.requestId, {
|
|
374
|
+
model: model.file,
|
|
375
|
+
verified: model.verified,
|
|
376
|
+
timerStartedAt: booking.timerStartedAt,
|
|
377
|
+
timerEndsAt: booking.timerEndsAt,
|
|
378
|
+
hours: booking.hours,
|
|
379
|
+
pricePerHourNanos: booking.pricePerHourNanos,
|
|
380
|
+
minTokS: booking.minTokS,
|
|
381
|
+
guaranteedNanos: booking.guaranteedNanos,
|
|
382
|
+
shared: Boolean(booking.shared),
|
|
383
|
+
tokS: booking.tokS && booking.tokS > 0 ? booking.tokS : null,
|
|
384
|
+
slots: Math.max(1, Math.round(Number(booking.slots) || 1)),
|
|
385
|
+
context: Number(booking.context) || context,
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
this.persistBookings();
|
|
389
|
+
this.syncDashboard({
|
|
390
|
+
wakeLock: this.sleepGuard.status().active,
|
|
391
|
+
status: 'working',
|
|
392
|
+
message: restorable.length === 1 ? `Resuming booking ${first.requestId} after a restart` : `Resuming ${restorable.length} bookings after a restart`,
|
|
393
|
+
});
|
|
394
|
+
for (const booking of restorable) {
|
|
395
|
+
this.logger.info('Resuming persisted booking', { requestId: booking.requestId, ends: isoTime(booking.timerEndsAt), slots: booking.slots });
|
|
396
|
+
}
|
|
397
|
+
return true;
|
|
398
|
+
}
|
|
399
|
+
catch (error) {
|
|
400
|
+
for (const booking of restorable)
|
|
401
|
+
this.removeBooking(booking.requestId, false);
|
|
402
|
+
// The wake lock may already be held for the bookings that could not be resumed.
|
|
403
|
+
await this.releaseWakeLockIfIdle();
|
|
404
|
+
// A stop during the restore (the load was aborted) is not a failed restore: stop() decides about state.json.
|
|
405
|
+
if (this.stopped)
|
|
406
|
+
return false;
|
|
407
|
+
this.logger.warn('Could not restore the previous booking', { error: error instanceof Error ? error.message : String(error) });
|
|
408
|
+
this.persisted = null;
|
|
409
|
+
await clearAgentState();
|
|
410
|
+
return false;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
/** Writes every booking to state.json (newest first), or removes the file when none is left. */
|
|
414
|
+
persistBookings() {
|
|
415
|
+
const model = this.model;
|
|
416
|
+
const list = [...this.bookings.values()].sort((a, b) => (b.timerStartedAt ?? 0) - (a.timerStartedAt ?? 0));
|
|
417
|
+
if (!model || !list.length) {
|
|
418
|
+
this.clearPersistedBooking();
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
const now = Date.now();
|
|
422
|
+
const saved = {
|
|
423
|
+
url: model.sourceUrl || model.path || '',
|
|
424
|
+
path: model.path || '',
|
|
425
|
+
file: model.file,
|
|
426
|
+
modelId: model.modelId,
|
|
427
|
+
label: model.label,
|
|
428
|
+
repo: model.repo,
|
|
429
|
+
quant: model.quant,
|
|
430
|
+
paramB: model.paramB,
|
|
431
|
+
layers: model.layers,
|
|
432
|
+
contextTrain: model.contextTrain,
|
|
433
|
+
sha256: model.sha256,
|
|
434
|
+
verified: model.verified,
|
|
435
|
+
needsMb: model.needsMb,
|
|
436
|
+
size: model.size,
|
|
437
|
+
architecture: model.architecture,
|
|
438
|
+
};
|
|
439
|
+
const records = list.map((booking) => ({
|
|
440
|
+
requestId: booking.requestId,
|
|
441
|
+
model: { ...saved },
|
|
442
|
+
context: booking.context,
|
|
443
|
+
// The slot count the booking was loaded with, so a restart starts llama-server the same way.
|
|
444
|
+
slots: booking.slots,
|
|
445
|
+
timerStartedAt: booking.timerStartedAt,
|
|
446
|
+
timerEndsAt: booking.timerEndsAt,
|
|
447
|
+
hours: booking.hours,
|
|
448
|
+
pricePerHourNanos: booking.pricePerHourNanos,
|
|
449
|
+
minTokS: booking.minTokS,
|
|
450
|
+
guaranteedNanos: booking.guaranteedNanos,
|
|
451
|
+
shared: booking.shared,
|
|
452
|
+
tokS: booking.tokS,
|
|
453
|
+
savedAt: now,
|
|
454
|
+
}));
|
|
455
|
+
// An older agent restores only `booking`, and drops it unless it is the server's `activeRequest`
|
|
456
|
+
// (the newest by claim time, which need not be the newest by start time). Write that one there.
|
|
457
|
+
const legacy = records.find((record) => record.requestId === this.serverNewestId) ?? records[0];
|
|
458
|
+
this.persisted = { version: 1, booking: legacy, bookings: records, updatedAt: now };
|
|
459
|
+
const snapshot = this.persisted;
|
|
460
|
+
this.stateWrite = this.stateWrite
|
|
461
|
+
.then(() => saveAgentState(snapshot))
|
|
462
|
+
.catch((error) => this.logger.warn('Could not save recovery state', { error: error instanceof Error ? error.message : String(error) }));
|
|
463
|
+
}
|
|
464
|
+
sortedBookings() {
|
|
465
|
+
return [...this.bookings.values()].sort((a, b) => (a.timerEndsAt ?? Number.MAX_SAFE_INTEGER) - (b.timerEndsAt ?? Number.MAX_SAFE_INTEGER));
|
|
466
|
+
}
|
|
467
|
+
/** Adds or updates a booking. A pending request with the same id becomes this booking. */
|
|
468
|
+
upsertBooking(requestId, fields) {
|
|
469
|
+
const pending = this.pending?.requestId === requestId ? this.pending : null;
|
|
470
|
+
let booking = this.bookings.get(requestId);
|
|
471
|
+
if (!booking) {
|
|
472
|
+
booking = {
|
|
473
|
+
requestId,
|
|
474
|
+
model: pending?.model ?? this.model?.file ?? null,
|
|
475
|
+
verified: pending?.verified ?? false,
|
|
476
|
+
timerStartedAt: null,
|
|
477
|
+
timerEndsAt: null,
|
|
478
|
+
hours: null,
|
|
479
|
+
pricePerHourNanos: null,
|
|
480
|
+
minTokS: null,
|
|
481
|
+
guaranteedNanos: null,
|
|
482
|
+
shared: false,
|
|
483
|
+
tokS: null,
|
|
484
|
+
slots: pending?.slots ?? this.capacity(),
|
|
485
|
+
context: pending?.context ?? this.runnerContext ?? this.options.context ?? 8192,
|
|
486
|
+
seenAt: Date.now(),
|
|
487
|
+
timer: null,
|
|
488
|
+
};
|
|
489
|
+
this.bookings.set(requestId, booking);
|
|
490
|
+
}
|
|
491
|
+
for (const [key, value] of Object.entries(fields)) {
|
|
492
|
+
if (value !== undefined)
|
|
493
|
+
booking[key] = value;
|
|
494
|
+
}
|
|
495
|
+
booking.seenAt = Date.now();
|
|
496
|
+
if (pending)
|
|
497
|
+
this.pending = null;
|
|
498
|
+
this.scheduleBookingEnd(booking);
|
|
499
|
+
return booking;
|
|
500
|
+
}
|
|
501
|
+
scheduleBookingEnd(booking) {
|
|
502
|
+
if (booking.timer)
|
|
503
|
+
clearTimeout(booking.timer);
|
|
504
|
+
booking.timer = null;
|
|
505
|
+
if (!booking.timerEndsAt)
|
|
506
|
+
return;
|
|
507
|
+
const delay = Math.max(0, booking.timerEndsAt - Date.now()) + 1_000;
|
|
508
|
+
booking.timer = setTimeout(() => this.finishBooking(booking.requestId), Math.min(delay, MAX_TIMER_MS));
|
|
509
|
+
}
|
|
510
|
+
/** The end timer of one booking fired: remove only that booking. */
|
|
511
|
+
finishBooking(requestId) {
|
|
512
|
+
const booking = this.bookings.get(requestId);
|
|
513
|
+
if (this.stopped || !booking)
|
|
514
|
+
return;
|
|
515
|
+
if (booking.timerEndsAt && Date.now() < booking.timerEndsAt) {
|
|
516
|
+
this.scheduleBookingEnd(booking);
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
this.logger.info('Booking window ended', { requestId });
|
|
520
|
+
this.removeBooking(requestId);
|
|
521
|
+
this.persistBookings();
|
|
522
|
+
void this.releaseWakeLockIfIdle();
|
|
523
|
+
const left = this.bookings.size;
|
|
524
|
+
this.syncDashboard({
|
|
525
|
+
status: left || this.activeJobs.size ? 'working' : 'online',
|
|
526
|
+
message: left ? `Booking ${requestId} finished; ${left} rental${left === 1 ? '' : 's'} still active.` : 'Booking finished. Back on standby.',
|
|
527
|
+
});
|
|
528
|
+
}
|
|
529
|
+
/**
|
|
530
|
+
* Removes one booking and books its earned share into the session total. A settled booking has
|
|
531
|
+
* ended: its id is remembered so a status answer built before the end cannot add it (and its
|
|
532
|
+
* earnings) a second time. `settle = false` only drops it (a restore that failed), so it is not remembered.
|
|
533
|
+
*/
|
|
534
|
+
removeBooking(requestId, settle = true) {
|
|
535
|
+
const booking = this.bookings.get(requestId);
|
|
536
|
+
if (!booking)
|
|
537
|
+
return null;
|
|
538
|
+
if (booking.timer)
|
|
539
|
+
clearTimeout(booking.timer);
|
|
540
|
+
booking.timer = null;
|
|
541
|
+
this.bookings.delete(requestId);
|
|
542
|
+
if (settle) {
|
|
543
|
+
const now = Date.now();
|
|
544
|
+
const share = this.bookingShare(booking, now);
|
|
545
|
+
this.settledEarnedNanos += share;
|
|
546
|
+
this.settledGuaranteedNanos += share;
|
|
547
|
+
for (const [id, endedAt] of this.endedIds)
|
|
548
|
+
if (now - endedAt > ENDED_TTL_MS)
|
|
549
|
+
this.endedIds.delete(id);
|
|
550
|
+
this.endedIds.set(requestId, now);
|
|
551
|
+
}
|
|
552
|
+
return booking;
|
|
553
|
+
}
|
|
554
|
+
clearBookingTimers() {
|
|
555
|
+
for (const booking of this.bookings.values()) {
|
|
556
|
+
if (booking.timer)
|
|
557
|
+
clearTimeout(booking.timer);
|
|
558
|
+
booking.timer = null;
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
/** The part of a booking's guaranteed pay that the time so far has earned. */
|
|
562
|
+
bookingShare(booking, now) {
|
|
563
|
+
if (!booking.guaranteedNanos || !booking.timerStartedAt || !booking.timerEndsAt)
|
|
564
|
+
return 0;
|
|
565
|
+
const total = Math.max(1, booking.timerEndsAt - booking.timerStartedAt);
|
|
566
|
+
const elapsed = Math.min(total, Math.max(0, now - booking.timerStartedAt));
|
|
567
|
+
return Math.round(booking.guaranteedNanos * (elapsed / total));
|
|
568
|
+
}
|
|
569
|
+
bookingJob(booking) {
|
|
570
|
+
return {
|
|
571
|
+
kind: 'request',
|
|
572
|
+
id: booking.requestId,
|
|
573
|
+
model: booking.model ?? this.model?.file ?? null,
|
|
574
|
+
verified: booking.verified,
|
|
575
|
+
timerStartedAt: booking.timerStartedAt,
|
|
576
|
+
timerEndsAt: booking.timerEndsAt,
|
|
577
|
+
minTokS: booking.minTokS,
|
|
578
|
+
guaranteedNanos: booking.guaranteedNanos,
|
|
579
|
+
hours: booking.hours,
|
|
580
|
+
shared: booking.shared,
|
|
581
|
+
tokS: booking.tokS,
|
|
582
|
+
};
|
|
583
|
+
}
|
|
584
|
+
baseJob() {
|
|
585
|
+
if (this.startupJob)
|
|
586
|
+
return { ...this.startupJob, model: this.model?.file || this.startupJob.model, verified: this.model?.verified ?? this.startupJob.verified };
|
|
587
|
+
if (this.options.publicMode || this.activeJobs.size > 0) {
|
|
588
|
+
return { kind: 'public', model: this.model?.file || null, verified: this.model?.verified, minTokS: null, guaranteedNanos: 0 };
|
|
589
|
+
}
|
|
590
|
+
return null;
|
|
591
|
+
}
|
|
592
|
+
/**
|
|
593
|
+
* Rebuilds the dashboard's request view from the bookings. Earnings are the per-token
|
|
594
|
+
* earnings plus the time-proportional share of every booking, active or settled.
|
|
595
|
+
*/
|
|
596
|
+
syncDashboard(extra = {}) {
|
|
597
|
+
if (this.stopped)
|
|
598
|
+
return;
|
|
599
|
+
const now = Date.now();
|
|
600
|
+
const bookings = this.sortedBookings();
|
|
601
|
+
const pending = this.pending;
|
|
602
|
+
const job = pending
|
|
603
|
+
? { kind: 'request', id: pending.requestId, model: pending.model, verified: pending.verified, timerStartedAt: null, timerEndsAt: null, minTokS: null, guaranteedNanos: 0 }
|
|
604
|
+
: bookings[0] ? this.bookingJob(bookings[0]) : this.baseJob();
|
|
605
|
+
let earned = this.tokenEarnedNanos + this.settledEarnedNanos;
|
|
606
|
+
let guaranteed = this.settledGuaranteedNanos;
|
|
607
|
+
for (const booking of bookings) {
|
|
608
|
+
earned += this.bookingShare(booking, now);
|
|
609
|
+
guaranteed += booking.guaranteedNanos || 0;
|
|
610
|
+
}
|
|
611
|
+
this.dashboard.update({
|
|
612
|
+
job,
|
|
613
|
+
bookings: bookings.map((booking) => this.bookingJob(booking)),
|
|
614
|
+
sessionEarnedNanos: earned,
|
|
615
|
+
sessionGuaranteedNanos: guaranteed,
|
|
616
|
+
tokenEarnedNanos: this.tokenEarnedNanos,
|
|
617
|
+
...extra,
|
|
618
|
+
});
|
|
619
|
+
}
|
|
620
|
+
/** Removes state.json if this agent wrote it; `force` also removes the file a restore in progress is working on. */
|
|
621
|
+
clearPersistedBooking(force = false) {
|
|
622
|
+
if (!this.persisted && !force)
|
|
623
|
+
return;
|
|
624
|
+
this.persisted = null;
|
|
625
|
+
this.stateWrite = this.stateWrite
|
|
626
|
+
.then(() => clearAgentState())
|
|
627
|
+
.catch(() => undefined);
|
|
628
|
+
}
|
|
629
|
+
async run() {
|
|
630
|
+
if (this.runStarted)
|
|
631
|
+
throw new Error('Agent.run() can only be called once');
|
|
632
|
+
this.runStarted = true;
|
|
633
|
+
if (this.stopped)
|
|
634
|
+
return;
|
|
635
|
+
const release = await acquireInstanceLock();
|
|
636
|
+
try {
|
|
637
|
+
await this.publishHealth();
|
|
638
|
+
this.healthTimer = setInterval(() => { void this.publishHealth(); }, 5_000);
|
|
639
|
+
await this.runOwned();
|
|
640
|
+
}
|
|
641
|
+
finally {
|
|
642
|
+
if (this.healthTimer)
|
|
643
|
+
clearInterval(this.healthTimer);
|
|
644
|
+
try {
|
|
645
|
+
const errors = [];
|
|
646
|
+
// A failed stop must not bypass pending handlers, late allocations, or
|
|
647
|
+
// final writes. Retain ownership until every cleanup step settles.
|
|
648
|
+
for (const cleanup of [
|
|
649
|
+
() => this.stop(true),
|
|
650
|
+
() => Promise.allSettled([...this.messageTasks]),
|
|
651
|
+
() => this.releaseResources(),
|
|
652
|
+
() => this.stateWrite,
|
|
653
|
+
() => this.publishHealth(),
|
|
654
|
+
]) {
|
|
655
|
+
try {
|
|
656
|
+
await cleanup();
|
|
657
|
+
}
|
|
658
|
+
catch (error) {
|
|
659
|
+
errors.push(error);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
// Preserve the first cleanup failure rather than masking it with a retry.
|
|
663
|
+
if (errors.length)
|
|
664
|
+
throw errors[0];
|
|
665
|
+
}
|
|
666
|
+
finally {
|
|
667
|
+
await release();
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
async runOwned() {
|
|
672
|
+
if (this.stopped)
|
|
673
|
+
return;
|
|
674
|
+
const cfg = this.options.config;
|
|
675
|
+
if (!cfg.token)
|
|
676
|
+
throw new Error('No host key saved. Run: sublera login');
|
|
677
|
+
this.logger.info(`Starting Sublera agent v${VERSION}`, { server: cfg.server, simulate: this.options.simulation });
|
|
678
|
+
if (!this.options.simulation) {
|
|
679
|
+
this.gpu = applyGpuOverride(await this.gpuMonitor.scan(true), this.options.gpuOverride);
|
|
680
|
+
for (const warning of this.gpu.warnings)
|
|
681
|
+
this.logger.warn(warning);
|
|
682
|
+
}
|
|
683
|
+
if (!this.gpu.primary && !this.options.simulation && !this.options.gpuOverride) {
|
|
684
|
+
this.logger.warn('No GPU detected. Use --gpu "RTX 4090" --vram 24576 for a test node, or --simulation.');
|
|
685
|
+
}
|
|
686
|
+
this.updateGpuDashboard();
|
|
687
|
+
this.cachedModels = await listCachedModels();
|
|
688
|
+
const restored = await this.restorePersistedBooking();
|
|
689
|
+
if (!restored && this.options.model && !this.stopped) {
|
|
690
|
+
try {
|
|
691
|
+
await this.prepareStartupModel(this.options.model);
|
|
692
|
+
}
|
|
693
|
+
catch (error) {
|
|
694
|
+
if (!this.stopped)
|
|
695
|
+
throw error;
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
if (this.stopped)
|
|
699
|
+
return this.releaseResources();
|
|
700
|
+
this.dashboard.start();
|
|
701
|
+
this.setupKeyboard();
|
|
702
|
+
this.dashboard.update({ status: 'starting', server: cfg.server });
|
|
703
|
+
const endpoint = await this.endpointFor(cfg);
|
|
704
|
+
this.dashboard.update({ endpoint });
|
|
705
|
+
this.tunnel = new TunnelClient({
|
|
706
|
+
server: cfg.server,
|
|
707
|
+
token: cfg.token,
|
|
708
|
+
wsPath: cfg.wsPath,
|
|
709
|
+
logger: this.logger,
|
|
710
|
+
hello: () => this.helloPayload(),
|
|
711
|
+
heartbeat: () => this.heartbeatPayload(),
|
|
712
|
+
});
|
|
713
|
+
this.tunnel.on('state', (state) => {
|
|
714
|
+
if (state !== 'open')
|
|
715
|
+
this.online = false;
|
|
716
|
+
if (state === 'open') {
|
|
717
|
+
this.dashboard.update({ status: this.activeJobs.size ? 'working' : 'online' });
|
|
718
|
+
}
|
|
719
|
+
else if (state === 'reconnecting') {
|
|
720
|
+
this.dashboard.update({ status: 'reconnecting', message: 'Tunnel dropped; keeping llama-server warm' });
|
|
721
|
+
}
|
|
722
|
+
else if (state === 'auth-failed') {
|
|
723
|
+
this.dashboard.update({ status: 'auth-failed', message: 'Host key rejected. Run sublera login again.' });
|
|
724
|
+
}
|
|
725
|
+
});
|
|
726
|
+
this.tunnel.on('outdated', (error) => { this.fail(error, 'outdated'); });
|
|
727
|
+
this.tunnel.on('message', (message) => {
|
|
728
|
+
const task = this.handleMessage(message).catch((error) => {
|
|
729
|
+
if (this.stopped)
|
|
730
|
+
return;
|
|
731
|
+
this.logger.error('Server message handling failed', { error: error instanceof Error ? error.message : String(error) });
|
|
732
|
+
this.fail(new Error('Server message handling failed'));
|
|
733
|
+
void this.stop(true);
|
|
734
|
+
});
|
|
735
|
+
this.messageTasks.add(task);
|
|
736
|
+
void task.finally(() => this.messageTasks.delete(task));
|
|
737
|
+
});
|
|
738
|
+
this.tunnel.on('auth-failed', (error) => { this.fail(error); });
|
|
739
|
+
this.tunnel.start();
|
|
740
|
+
try {
|
|
741
|
+
await this.tunnel.waitForWelcome(20_000);
|
|
742
|
+
}
|
|
743
|
+
catch (error) {
|
|
744
|
+
if (this.stopped)
|
|
745
|
+
return this.releaseResources();
|
|
746
|
+
this.dashboard.update({ status: 'offline', message: error instanceof Error ? error.message : String(error) });
|
|
747
|
+
await this.stop(true);
|
|
748
|
+
throw error;
|
|
749
|
+
}
|
|
750
|
+
this.dashboard.update({ status: 'online', endpoint, message: null });
|
|
751
|
+
await this.refreshMe();
|
|
752
|
+
if (this.stopped)
|
|
753
|
+
return this.releaseResources();
|
|
754
|
+
this.markOnline();
|
|
755
|
+
setTimeout(() => { void this.refreshMe(); }, 1_200);
|
|
756
|
+
this.meTimer = setInterval(() => { void this.refreshMe(); }, 10_000);
|
|
757
|
+
this.startGpuPolling();
|
|
758
|
+
this.logger.info('Agent is online. Waiting for work.');
|
|
759
|
+
if (this.options.once) {
|
|
760
|
+
await new Promise((resolve) => this.once('once', resolve));
|
|
761
|
+
await this.stop();
|
|
762
|
+
return;
|
|
763
|
+
}
|
|
764
|
+
if (this.stopped)
|
|
765
|
+
return;
|
|
766
|
+
await new Promise((resolve) => {
|
|
767
|
+
this.once('stop', resolve);
|
|
768
|
+
});
|
|
769
|
+
}
|
|
770
|
+
async endpointFor(cfg) {
|
|
771
|
+
return hostEndpoint(cfg.server, this.me?.user.username);
|
|
772
|
+
}
|
|
773
|
+
/** Tells the server about a newly downloaded model, so the board can show it as ready on this host. */
|
|
774
|
+
async refreshCachedModels() {
|
|
775
|
+
this.cachedModels = await listCachedModels();
|
|
776
|
+
this.tunnel?.send({ type: 'models', cachedModels: this.cachedModels });
|
|
777
|
+
}
|
|
778
|
+
async prepareStartupModel(input) {
|
|
779
|
+
const inspection = this.options.modelInfo
|
|
780
|
+
|| (this.options.simulation || !/^https?:\/\//i.test(input)
|
|
781
|
+
? null
|
|
782
|
+
: await inspectHuggingFace(input).catch(() => null));
|
|
783
|
+
const loaded = await this.loadModel(input, {
|
|
784
|
+
context: this.options.context ?? 8192,
|
|
785
|
+
capContext: true,
|
|
786
|
+
requestId: null,
|
|
787
|
+
expectedSha: inspection?.sha256 ?? null,
|
|
788
|
+
expectedLayers: inspection?.layers ?? null,
|
|
789
|
+
file: inspection?.file ?? null,
|
|
790
|
+
});
|
|
791
|
+
this.model = loaded;
|
|
792
|
+
this.startupJob = {
|
|
793
|
+
kind: this.options.publicMode ? 'public' : 'request',
|
|
794
|
+
model: loaded.file,
|
|
795
|
+
verified: loaded.verified,
|
|
796
|
+
minTokS: null,
|
|
797
|
+
guaranteedNanos: 0,
|
|
798
|
+
};
|
|
799
|
+
this.syncDashboard({ llamaStatus: this.runner?.status || 'ready' });
|
|
800
|
+
}
|
|
801
|
+
async loadModel(input, opts) {
|
|
802
|
+
if (this.options.simulation) {
|
|
803
|
+
const file = opts.file || (input.startsWith('http') ? parseHfUrl(input)?.file : basename(input)) || 'sublera-simulator.gguf';
|
|
804
|
+
const paramB = paramBFromName(file) || 7;
|
|
805
|
+
const quant = quantFromName(file) || 'Q4_K_M';
|
|
806
|
+
return {
|
|
807
|
+
path: null,
|
|
808
|
+
sourceUrl: input,
|
|
809
|
+
file,
|
|
810
|
+
modelId: file.replace(/\.gguf$/i, ''),
|
|
811
|
+
label: file.replace(/\.gguf$/i, ''),
|
|
812
|
+
repo: parseHfUrl(input)?.repo || '',
|
|
813
|
+
quant,
|
|
814
|
+
paramB,
|
|
815
|
+
layers: opts.expectedLayers ?? null,
|
|
816
|
+
contextTrain: opts.context,
|
|
817
|
+
sha256: opts.expectedSha ?? null,
|
|
818
|
+
verified: false,
|
|
819
|
+
needsMb: estimateVramMb({ paramB, quant, context: opts.context, layers: opts.expectedLayers ?? undefined, slots: this.runnerSlots() }),
|
|
820
|
+
size: 0,
|
|
821
|
+
architecture: 'simulation',
|
|
822
|
+
};
|
|
823
|
+
}
|
|
824
|
+
let loaded = null;
|
|
825
|
+
if (/^https?:\/\//i.test(input)) {
|
|
826
|
+
loaded = await downloadModel({
|
|
827
|
+
url: input,
|
|
828
|
+
expectedSha256: opts.expectedSha ?? null,
|
|
829
|
+
expectedLayers: opts.expectedLayers ?? null,
|
|
830
|
+
context: opts.context,
|
|
831
|
+
slots: this.runnerSlots(),
|
|
832
|
+
// A file already on disk is hashed again: the SHA-256 the server gets is of the bytes that will be served.
|
|
833
|
+
verifyCached: true,
|
|
834
|
+
onProgress: (progress) => {
|
|
835
|
+
if (opts.requestId) {
|
|
836
|
+
this.tunnel?.send({ type: 'progress', requestId: opts.requestId, note: progress.note });
|
|
837
|
+
}
|
|
838
|
+
this.dashboard.update({ message: progress.note });
|
|
839
|
+
},
|
|
840
|
+
showProgress: true,
|
|
841
|
+
signal: opts.signal ? anySignal([this.startup.signal, opts.signal]) : this.startup.signal,
|
|
842
|
+
});
|
|
843
|
+
await this.refreshCachedModels();
|
|
844
|
+
}
|
|
845
|
+
else {
|
|
846
|
+
loaded = await verifyModelFile(input, { sha256: opts.expectedSha ?? null, layers: opts.expectedLayers ?? null, context: opts.context, slots: this.runnerSlots() });
|
|
847
|
+
}
|
|
848
|
+
// llama-server never gives a slot more context than the model was trained for, so check before starting it.
|
|
849
|
+
let context = opts.context;
|
|
850
|
+
if (loaded.contextTrain && loaded.contextTrain < context) {
|
|
851
|
+
const limit = `This model supports at most ${loaded.contextTrain} tokens of context`;
|
|
852
|
+
if (!opts.capContext)
|
|
853
|
+
throw new Error(`${limit}, but ${context} were asked for. Ask for ${loaded.contextTrain} or less, or pick another model.`);
|
|
854
|
+
this.logger.warn(`${limit}; using ${loaded.contextTrain} instead of ${context}.`);
|
|
855
|
+
context = loaded.contextTrain;
|
|
856
|
+
}
|
|
857
|
+
const needsMb = loaded.needsMb ?? estimateVramMb({
|
|
858
|
+
paramB: loaded.paramB || 7,
|
|
859
|
+
quant: loaded.quant || 'Q4_K_M',
|
|
860
|
+
context,
|
|
861
|
+
layers: loaded.layers ?? undefined,
|
|
862
|
+
slots: this.runnerSlots(),
|
|
863
|
+
});
|
|
864
|
+
// When llama-server already runs this file with these settings, its memory is part of the
|
|
865
|
+
// "used" VRAM, so a fit check would count the model twice. Reuse it without checking.
|
|
866
|
+
const plan = this.runnerPlan(loaded.path, context, this.runnerSlots(), opts.requestId);
|
|
867
|
+
if (plan.action === 'refuse')
|
|
868
|
+
throw new Error(`${plan.note}. Retry when that work ends.`);
|
|
869
|
+
if (plan.action !== 'reuse') {
|
|
870
|
+
const fit = checkFit(needsMb, this.gpu.primary);
|
|
871
|
+
if (this.gpu.primary && !fit.ok)
|
|
872
|
+
throw new Error(fit.reason || 'Model does not fit in free VRAM');
|
|
873
|
+
}
|
|
874
|
+
const model = {
|
|
875
|
+
path: loaded.path,
|
|
876
|
+
sourceUrl: input,
|
|
877
|
+
file: loaded.file,
|
|
878
|
+
modelId: loaded.info.name || loaded.file.replace(/\.gguf$/i, ''),
|
|
879
|
+
label: loaded.info.name || loaded.file.replace(/\.gguf$/i, ''),
|
|
880
|
+
repo: parseHfUrl(input)?.repo || '',
|
|
881
|
+
quant: loaded.quant || 'Q4_K_M',
|
|
882
|
+
paramB: loaded.paramB,
|
|
883
|
+
layers: loaded.layers,
|
|
884
|
+
contextTrain: loaded.contextTrain,
|
|
885
|
+
sha256: loaded.sha256,
|
|
886
|
+
verified: loaded.verified,
|
|
887
|
+
needsMb,
|
|
888
|
+
size: loaded.size,
|
|
889
|
+
architecture: loaded.architecture,
|
|
890
|
+
};
|
|
891
|
+
await this.ensureRunner(model, context, this.runnerSlots(), opts.requestId);
|
|
892
|
+
return model;
|
|
893
|
+
}
|
|
894
|
+
/**
|
|
895
|
+
* True while llama-server serves someone other than `owner`: an inference job, another
|
|
896
|
+
* booking, public selling, or another request whose model is loaded and in its model check
|
|
897
|
+
* (or a benchmark). A restart would cut them off, so it is not allowed then.
|
|
898
|
+
*/
|
|
899
|
+
runnerBusy(owner) {
|
|
900
|
+
if (this.activeJobs.size > 0)
|
|
901
|
+
return true;
|
|
902
|
+
if (this.options.publicMode && this.model)
|
|
903
|
+
return true;
|
|
904
|
+
for (const id of this.bookings.keys())
|
|
905
|
+
if (id !== owner)
|
|
906
|
+
return true;
|
|
907
|
+
const checking = this.checkingRequest();
|
|
908
|
+
return Boolean(checking && checking.requestId !== owner);
|
|
909
|
+
}
|
|
910
|
+
/** Decides whether loading `path` reuses, restarts, or must not touch the running llama-server. */
|
|
911
|
+
runnerPlan(path, context, slots, owner) {
|
|
912
|
+
// The simulator has no engine to restart.
|
|
913
|
+
if (this.options.simulation)
|
|
914
|
+
return { action: 'reuse' };
|
|
915
|
+
const runner = this.runner;
|
|
916
|
+
if (!runner || !path || runner.status === 'stopped' || runner.status === 'failed')
|
|
917
|
+
return { action: 'start' };
|
|
918
|
+
const samePath = this.runnerModelPath === path;
|
|
919
|
+
if (samePath && runner.status === 'ready' && this.runnerContext === context && this.runnerSlotCount === slots)
|
|
920
|
+
return { action: 'reuse' };
|
|
921
|
+
if (!this.runnerBusy(owner))
|
|
922
|
+
return { action: 'restart' };
|
|
923
|
+
if (!samePath) {
|
|
924
|
+
return { action: 'refuse', note: `llama-server is serving other work with ${basename(this.runnerModelPath || 'another model')}, so it cannot load another model now` };
|
|
925
|
+
}
|
|
926
|
+
if ((this.runnerContext ?? 0) < context) {
|
|
927
|
+
return { action: 'refuse', note: `llama-server is serving other work with ${this.runnerContext} tokens of context per slot, so it cannot restart for ${context}` };
|
|
928
|
+
}
|
|
929
|
+
if (this.runnerSlotCount !== slots) {
|
|
930
|
+
return { action: 'reuse', note: `Keeping llama-server at ${this.runnerSlotCount} parallel slots (${slots} asked for) because it is serving other work` };
|
|
931
|
+
}
|
|
932
|
+
return { action: 'reuse' };
|
|
933
|
+
}
|
|
934
|
+
/** Requests are served on the slots llama-server really runs with. */
|
|
935
|
+
capacity() {
|
|
936
|
+
return this.runner && this.runnerSlotCount ? this.runnerSlotCount : this.runnerSlots();
|
|
937
|
+
}
|
|
938
|
+
async ensureRunner(model, context, slots = this.runnerSlots(), owner = null) {
|
|
939
|
+
if (this.options.simulation || !model.path)
|
|
940
|
+
return;
|
|
941
|
+
if (!this.install) {
|
|
942
|
+
const result = this.options.engine
|
|
943
|
+
? { found: true, install: { path: this.options.engine.binaryPath, version: 'test', variant: 'cpu', sha256: null, verified: false, source: 'env' } }
|
|
944
|
+
: await ensureLlamaServer({
|
|
945
|
+
logger: this.logger,
|
|
946
|
+
gpu: this.gpu,
|
|
947
|
+
fix: this.options.autoInstall !== false,
|
|
948
|
+
requireChecksum: this.options.requireChecksum,
|
|
949
|
+
showProgress: true,
|
|
950
|
+
});
|
|
951
|
+
if (!result.found)
|
|
952
|
+
throw new Error(result.reason);
|
|
953
|
+
this.install = result.install;
|
|
954
|
+
this.logger.info('llama-server ready', { path: result.install.path, variant: result.install.variant, verified: result.install.verified });
|
|
955
|
+
const source = result.install.source;
|
|
956
|
+
if (source === 'env' || source === 'path') {
|
|
957
|
+
this.logger.warn(`Using your own llama-server (${source === 'env' ? 'set in LLAMA_SERVER' : 'found on your PATH'}), not the build Sublera installs. `
|
|
958
|
+
+ 'The server compares your answers with other hosts; a different llama.cpp build can fail those model checks. '
|
|
959
|
+
+ `To use the official build, ${source === 'env' ? 'unset LLAMA_SERVER' : 'take llama-server off your PATH'} and run: sublera doctor --fix`);
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
const plan = this.runnerPlan(model.path, context, slots, owner);
|
|
963
|
+
if (plan.action === 'refuse')
|
|
964
|
+
throw new Error(`${plan.note}. Retry when that work ends.`);
|
|
965
|
+
if (plan.action === 'reuse') {
|
|
966
|
+
if (plan.note)
|
|
967
|
+
this.logger.warn(plan.note);
|
|
968
|
+
return;
|
|
969
|
+
}
|
|
970
|
+
if (this.runner) {
|
|
971
|
+
await this.runner.stop();
|
|
972
|
+
this.runner = null;
|
|
973
|
+
this.runnerModelPath = null;
|
|
974
|
+
this.runnerContext = null;
|
|
975
|
+
this.runnerSlotCount = null;
|
|
976
|
+
}
|
|
977
|
+
const runner = new LlamaRunner({
|
|
978
|
+
binaryPath: this.install.path,
|
|
979
|
+
modelPath: model.path,
|
|
980
|
+
context,
|
|
981
|
+
slots,
|
|
982
|
+
baseArgs: this.options.engine?.baseArgs,
|
|
983
|
+
env: this.options.engine?.env,
|
|
984
|
+
logger: this.logger,
|
|
985
|
+
restart: { maxRestarts: 5, windowMs: 120_000, delayMs: 500 },
|
|
986
|
+
onLog: (line) => {
|
|
987
|
+
this.logLines.push(line);
|
|
988
|
+
if (this.logLines.length > 200)
|
|
989
|
+
this.logLines.shift();
|
|
990
|
+
if (this.dashboard.current.showLogs)
|
|
991
|
+
this.dashboard.update({ logTail: this.logLines.slice(-5) });
|
|
992
|
+
},
|
|
993
|
+
});
|
|
994
|
+
runner.on('status', (status) => {
|
|
995
|
+
this.dashboard.update({ llamaStatus: status });
|
|
996
|
+
});
|
|
997
|
+
runner.on('ready', () => {
|
|
998
|
+
this.dashboard.update({ llamaStatus: 'ready' });
|
|
999
|
+
// After a crash the engine reloads the model; make sure it still answers as it did before.
|
|
1000
|
+
if (runner.restarts > 0 && this.runner === runner)
|
|
1001
|
+
void this.selfCheckEngine('restart');
|
|
1002
|
+
});
|
|
1003
|
+
runner.on('crash', () => {
|
|
1004
|
+
this.logger.warn('llama-server crashed; the runner is restarting it');
|
|
1005
|
+
this.dashboard.update({ message: 'llama-server crashed; restarting in place' });
|
|
1006
|
+
});
|
|
1007
|
+
runner.on('fatal', (error) => {
|
|
1008
|
+
this.logger.error('llama-server could not recover', { error: error.message });
|
|
1009
|
+
this.dashboard.update({ llamaStatus: 'failed', message: error.message });
|
|
1010
|
+
});
|
|
1011
|
+
this.runner = runner;
|
|
1012
|
+
await runner.start();
|
|
1013
|
+
this.runnerModelPath = model.path;
|
|
1014
|
+
this.runnerContext = context;
|
|
1015
|
+
this.runnerSlotCount = slots;
|
|
1016
|
+
this.dashboard.update({ llamaStatus: runner.status, gpuName: this.gpu.primary?.name || null });
|
|
1017
|
+
await this.selfCheckEngine('load');
|
|
1018
|
+
}
|
|
1019
|
+
/**
|
|
1020
|
+
* Asks the engine one short fixed question and compares the answer's token probabilities:
|
|
1021
|
+
* twice in a row after a load, and against the load-time answer after a crash restart. It only
|
|
1022
|
+
* warns; the server's model checks decide. It runs when the engine's prompt cache is empty and
|
|
1023
|
+
* no buyer is waiting, and a buyer's job cancels it.
|
|
1024
|
+
*/
|
|
1025
|
+
async selfCheckEngine(reason) {
|
|
1026
|
+
const runner = this.runner;
|
|
1027
|
+
const modelPath = this.runnerModelPath;
|
|
1028
|
+
if (this.options.simulation || !runner || !modelPath || runner.status !== 'ready' || this.activeJobs.size > 0 || this.stopped)
|
|
1029
|
+
return;
|
|
1030
|
+
this.selfCheckAbort?.abort();
|
|
1031
|
+
const controller = new AbortController();
|
|
1032
|
+
this.selfCheckAbort = controller;
|
|
1033
|
+
const signal = AbortSignal.any([controller.signal, this.startup.signal]);
|
|
1034
|
+
try {
|
|
1035
|
+
const first = await probeEngine(runner.url, { signal });
|
|
1036
|
+
if (!first) {
|
|
1037
|
+
this.logger.debug('Engine self-check skipped: the engine gave no token probabilities in time');
|
|
1038
|
+
return;
|
|
1039
|
+
}
|
|
1040
|
+
if (signal.aborted || this.stopped || this.runner !== runner)
|
|
1041
|
+
return;
|
|
1042
|
+
const baseline = this.engineBaseline?.modelPath === modelPath ? this.engineBaseline.fingerprint : null;
|
|
1043
|
+
if (reason === 'restart' && baseline) {
|
|
1044
|
+
const diff = compareFingerprints(baseline, first);
|
|
1045
|
+
if (diff.same === null) {
|
|
1046
|
+
this.logger.debug('Engine self-check inconclusive after restart', { reason: diff.reason });
|
|
1047
|
+
return;
|
|
1048
|
+
}
|
|
1049
|
+
if (diff.same === true) {
|
|
1050
|
+
this.logger.info('Engine self-check passed after the restart', { tokens: diff.compared });
|
|
1051
|
+
return;
|
|
1052
|
+
}
|
|
1053
|
+
const note = `Self-check: after the restart llama-server answers differently than when the model was loaded (${diff.reason}). The server's model checks may fail.`;
|
|
1054
|
+
this.logger.warn(note);
|
|
1055
|
+
this.dashboard.update({ message: note });
|
|
1056
|
+
return;
|
|
1057
|
+
}
|
|
1058
|
+
const second = await probeEngine(runner.url, { signal });
|
|
1059
|
+
if (!second || signal.aborted || this.stopped || this.runner !== runner)
|
|
1060
|
+
return;
|
|
1061
|
+
const diff = compareFingerprints(first, second);
|
|
1062
|
+
if (diff.same === null) {
|
|
1063
|
+
this.logger.debug('Engine self-check inconclusive', { reason: diff.reason });
|
|
1064
|
+
return;
|
|
1065
|
+
}
|
|
1066
|
+
if (diff.same === true) {
|
|
1067
|
+
this.engineBaseline = { modelPath, fingerprint: first };
|
|
1068
|
+
this.logger.info('Engine self-check passed: the same prompt gave the same answer twice', { tokens: diff.compared });
|
|
1069
|
+
return;
|
|
1070
|
+
}
|
|
1071
|
+
const note = `Self-check: the engine returned different token probabilities for the same prompt (${diff.reason}). This is diagnostic only, not proof of a fake model or faulty GPU. Check engine settings and repeat the test.`;
|
|
1072
|
+
this.logger.warn(note);
|
|
1073
|
+
this.dashboard.update({ message: note });
|
|
1074
|
+
}
|
|
1075
|
+
catch (error) {
|
|
1076
|
+
this.logger.debug('Engine self-check could not run', { error: error instanceof Error ? error.message : String(error) });
|
|
1077
|
+
}
|
|
1078
|
+
finally {
|
|
1079
|
+
if (this.selfCheckAbort === controller)
|
|
1080
|
+
this.selfCheckAbort = null;
|
|
1081
|
+
}
|
|
1082
|
+
}
|
|
1083
|
+
runnerSlots() {
|
|
1084
|
+
if (this.options.publicMode)
|
|
1085
|
+
return Math.max(1, this.options.slots ?? 4);
|
|
1086
|
+
return Math.max(1, this.options.slots ?? 1, this.bookingSlots);
|
|
1087
|
+
}
|
|
1088
|
+
helloPayload() {
|
|
1089
|
+
const gpu = this.gpu.primary;
|
|
1090
|
+
const model = this.model;
|
|
1091
|
+
const advertised = this.options.simulation ? (this.options.tokPerSec || 24) : (this.lastSpeed || this.options.tokPerSec || 24);
|
|
1092
|
+
return {
|
|
1093
|
+
type: 'hello',
|
|
1094
|
+
requestId: this.options.requestId ?? null,
|
|
1095
|
+
cachedModels: this.cachedModels,
|
|
1096
|
+
agentVersion: VERSION,
|
|
1097
|
+
platform: process.platform,
|
|
1098
|
+
arch: process.arch,
|
|
1099
|
+
gpuName: gpu?.name || this.options.gpuOverride?.name || 'Unknown GPU',
|
|
1100
|
+
gpuVendor: gpu?.vendor || 'unknown',
|
|
1101
|
+
vramMb: gpu?.totalVramMb || this.options.gpuOverride?.vramMb || 0,
|
|
1102
|
+
freeVramMb: gpu?.freeVramMb || 0,
|
|
1103
|
+
modelId: model?.modelId || (this.options.publicMode ? 'sublet-public' : 'sublet-standby'),
|
|
1104
|
+
modelLabel: model?.label || (this.options.publicMode ? 'Sublera public host' : 'Standby'),
|
|
1105
|
+
repo: model?.repo || '',
|
|
1106
|
+
file: model?.file || '',
|
|
1107
|
+
quant: model?.quant || 'Q4_K_M',
|
|
1108
|
+
paramB: model?.paramB ?? null,
|
|
1109
|
+
layers: model?.layers ?? null,
|
|
1110
|
+
context: this.runnerContext ?? this.sortedBookings()[0]?.context ?? this.options.context ?? 8192,
|
|
1111
|
+
slots: this.capacity(),
|
|
1112
|
+
sha256: model?.sha256 || null,
|
|
1113
|
+
hashVerified: Boolean(model?.verified),
|
|
1114
|
+
tokPerSec: advertised,
|
|
1115
|
+
ttftMs: this.options.simulation ? 300 : (this.lastTtft || 300),
|
|
1116
|
+
priceInUsd: this.options.priceInUsd ?? (this.options.price != null ? Math.max(0.01, this.options.price / 2) : 0.1),
|
|
1117
|
+
priceOutUsd: this.priceOutUsd(),
|
|
1118
|
+
priceHourUsd: this.priceHourUsd(),
|
|
1119
|
+
simulate: this.options.simulation,
|
|
1120
|
+
publicMode: Boolean(this.options.publicMode),
|
|
1121
|
+
};
|
|
1122
|
+
}
|
|
1123
|
+
heartbeatPayload() {
|
|
1124
|
+
if (!this.tunnel?.connected)
|
|
1125
|
+
return null;
|
|
1126
|
+
const gpu = this.gpu.primary;
|
|
1127
|
+
const active = [...this.activeJobs.values()][0];
|
|
1128
|
+
this.heartbeatAt = Date.now();
|
|
1129
|
+
const advertised = this.options.simulation ? (this.options.tokPerSec || 24) : (this.lastSpeed || this.options.tokPerSec || 24);
|
|
1130
|
+
return {
|
|
1131
|
+
type: 'heartbeat',
|
|
1132
|
+
tokPerSec: advertised,
|
|
1133
|
+
ttftMs: this.options.simulation ? 300 : (this.lastTtft || 300),
|
|
1134
|
+
llamaUp: this.options.simulation || this.runner?.status === 'ready',
|
|
1135
|
+
sha256: this.model?.sha256 || null,
|
|
1136
|
+
file: this.model?.file || '',
|
|
1137
|
+
layers: this.model?.layers ?? null,
|
|
1138
|
+
activeJobId: active?.jobId || null,
|
|
1139
|
+
wakeLock: this.sleepGuard.status().active,
|
|
1140
|
+
gpuTempC: gpu?.temperatureC ?? null,
|
|
1141
|
+
gpuPowerW: gpu?.powerW ?? null,
|
|
1142
|
+
gpuUtilPct: gpu?.utilizationPct ?? null,
|
|
1143
|
+
freeVramMb: gpu?.freeVramMb ?? 0,
|
|
1144
|
+
slotsUsed: this.activeJobs.size,
|
|
1145
|
+
};
|
|
1146
|
+
}
|
|
1147
|
+
async handleMessage(message) {
|
|
1148
|
+
if (this.stopped)
|
|
1149
|
+
return;
|
|
1150
|
+
// The requestId ends up in dashboard text, logs and replies, so only the server's id shape is accepted.
|
|
1151
|
+
if (REQUEST_FRAMES.has(message.type) && !isRequestId(message.requestId)) {
|
|
1152
|
+
this.logger.debug('Ignoring a server frame with a malformed requestId', { type: message.type });
|
|
1153
|
+
return;
|
|
1154
|
+
}
|
|
1155
|
+
switch (message.type) {
|
|
1156
|
+
case 'welcome':
|
|
1157
|
+
if (!this.draining)
|
|
1158
|
+
this.markOnline();
|
|
1159
|
+
this.dashboard.update({ status: 'online' });
|
|
1160
|
+
this.emit('welcome');
|
|
1161
|
+
return;
|
|
1162
|
+
case 'infer':
|
|
1163
|
+
await this.handleInfer(message);
|
|
1164
|
+
return;
|
|
1165
|
+
case 'load':
|
|
1166
|
+
if (this.draining) {
|
|
1167
|
+
this.tunnel?.send({ type: 'load_failed', requestId: message.requestId, message: 'Agent draining; retry on another host' });
|
|
1168
|
+
return;
|
|
1169
|
+
}
|
|
1170
|
+
await this.handleLoad(message);
|
|
1171
|
+
return;
|
|
1172
|
+
case 'cancel': {
|
|
1173
|
+
const job = this.activeJobs.get(String(message.jobId));
|
|
1174
|
+
if (job)
|
|
1175
|
+
job.controller.abort(new Error('Cancelled by server'));
|
|
1176
|
+
return;
|
|
1177
|
+
}
|
|
1178
|
+
case 'verified': {
|
|
1179
|
+
const requestId = String(message.requestId || '');
|
|
1180
|
+
const pending = this.pending;
|
|
1181
|
+
if (pending?.requestId === requestId) {
|
|
1182
|
+
// A check result that arrives while the model is still loading belongs to an older load.
|
|
1183
|
+
if (!pending.loaded)
|
|
1184
|
+
return;
|
|
1185
|
+
pending.verified = true;
|
|
1186
|
+
this.syncDashboard({ message: 'Model check passed. The clock can start.' });
|
|
1187
|
+
return;
|
|
1188
|
+
}
|
|
1189
|
+
const booking = this.bookings.get(requestId);
|
|
1190
|
+
if (booking) {
|
|
1191
|
+
booking.verified = true;
|
|
1192
|
+
this.syncDashboard();
|
|
1193
|
+
return;
|
|
1194
|
+
}
|
|
1195
|
+
// A seat or hourly rental on the model that is already loaded: its booking frame follows.
|
|
1196
|
+
this.logger.debug('Model check passed for a request without a load', { requestId });
|
|
1197
|
+
return;
|
|
1198
|
+
}
|
|
1199
|
+
case 'booking':
|
|
1200
|
+
this.applyBooking(message);
|
|
1201
|
+
return;
|
|
1202
|
+
case 'reject': {
|
|
1203
|
+
const requestId = String(message.requestId || '');
|
|
1204
|
+
// Shown on the dashboard and printed by start/jobs: no control characters from the server.
|
|
1205
|
+
const reason = printable(String(message.reason || 'unknown'));
|
|
1206
|
+
const booking = this.bookings.get(requestId) ?? null;
|
|
1207
|
+
const pending = this.pending?.requestId === requestId;
|
|
1208
|
+
const loading = this.loadAbort?.requestId === requestId;
|
|
1209
|
+
if (!booking && !pending && !loading) {
|
|
1210
|
+
this.emit('rejected', { requestId, reason, wasActive: false });
|
|
1211
|
+
return;
|
|
1212
|
+
}
|
|
1213
|
+
// A reject ends only this request; other bookings keep running.
|
|
1214
|
+
if (loading) {
|
|
1215
|
+
this.loadAbort.controller.abort(new Error(`Request ${requestId} stopped: ${reason}`));
|
|
1216
|
+
this.loadAbort = null;
|
|
1217
|
+
}
|
|
1218
|
+
if (pending)
|
|
1219
|
+
this.pending = null;
|
|
1220
|
+
if (booking) {
|
|
1221
|
+
this.removeBooking(requestId);
|
|
1222
|
+
this.persistBookings();
|
|
1223
|
+
}
|
|
1224
|
+
void this.releaseWakeLockIfIdle();
|
|
1225
|
+
this.syncDashboard({
|
|
1226
|
+
status: this.bookings.size || this.activeJobs.size ? 'working' : 'online',
|
|
1227
|
+
message: requestId.startsWith('bench_') ? reason : `Request ${requestId || '?'} stopped: ${reason}`,
|
|
1228
|
+
});
|
|
1229
|
+
this.emit('rejected', { requestId, reason, wasActive: Boolean(booking?.timerEndsAt) });
|
|
1230
|
+
return;
|
|
1231
|
+
}
|
|
1232
|
+
case 'error':
|
|
1233
|
+
this.logger.warn(printable(String(message.message || 'Server error')));
|
|
1234
|
+
return;
|
|
1235
|
+
default:
|
|
1236
|
+
this.logger.debug('Unhandled server message', { type: String(message.type) });
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
/**
|
|
1240
|
+
* The server started a booking's clock. It may be for the request just loaded, or for a
|
|
1241
|
+
* request that never had a load frame (a speed seat or hourly rental on the loaded model).
|
|
1242
|
+
* Each booking keeps its own end timer; a model load in progress stays busy.
|
|
1243
|
+
*/
|
|
1244
|
+
applyBooking(message) {
|
|
1245
|
+
const requestId = message.requestId;
|
|
1246
|
+
if (!isRequestId(requestId)) {
|
|
1247
|
+
this.logger.debug('Ignoring a booking frame without a valid requestId');
|
|
1248
|
+
return;
|
|
1249
|
+
}
|
|
1250
|
+
if (this.endedIds.has(requestId)) {
|
|
1251
|
+
// Its earned share is already settled; adding it again would count it twice.
|
|
1252
|
+
this.logger.debug('Ignoring a booking frame for a booking that already ended', { requestId });
|
|
1253
|
+
return;
|
|
1254
|
+
}
|
|
1255
|
+
const timerEndsAt = validTime(message.timerEndsAt);
|
|
1256
|
+
const timerStartedAt = validTime(message.timerStartedAt) ?? Date.now();
|
|
1257
|
+
const hours = finiteNumber(message.hours) || (timerEndsAt ? (timerEndsAt - timerStartedAt) / 3_600_000 : null);
|
|
1258
|
+
const minTokS = finiteNumber(message.minTokS) || null;
|
|
1259
|
+
const pricePerHour = finiteNumber(message.pricePerHourNanos) || null;
|
|
1260
|
+
const guaranteed = hours && pricePerHour ? Math.round(hours * pricePerHour) : null;
|
|
1261
|
+
const seatTokS = finiteNumber(message.tokS);
|
|
1262
|
+
const tokS = seatTokS != null && seatTokS > 0 ? seatTokS : null;
|
|
1263
|
+
const booking = this.upsertBooking(requestId, {
|
|
1264
|
+
verified: true,
|
|
1265
|
+
timerStartedAt,
|
|
1266
|
+
timerEndsAt,
|
|
1267
|
+
hours,
|
|
1268
|
+
pricePerHourNanos: pricePerHour,
|
|
1269
|
+
minTokS,
|
|
1270
|
+
guaranteedNanos: guaranteed,
|
|
1271
|
+
shared: message.shared === true || tokS != null,
|
|
1272
|
+
tokS,
|
|
1273
|
+
});
|
|
1274
|
+
this.holdWakeLock();
|
|
1275
|
+
const count = this.bookings.size;
|
|
1276
|
+
this.logger.info('Booking started', { requestId, ends: isoTime(timerEndsAt), tokS: booking.tokS ?? 'full GPU', active: count });
|
|
1277
|
+
this.syncDashboard({
|
|
1278
|
+
status: 'working',
|
|
1279
|
+
wakeLock: this.sleepGuard.status().active,
|
|
1280
|
+
message: count > 1 ? `Booking ${requestId} started; ${count} rentals active.` : 'Booking timer started. Keep the terminal open.',
|
|
1281
|
+
});
|
|
1282
|
+
this.persistBookings();
|
|
1283
|
+
}
|
|
1284
|
+
async handleInfer(message) {
|
|
1285
|
+
const jobId = String(message.jobId || '');
|
|
1286
|
+
if (!jobId || this.stopped)
|
|
1287
|
+
return;
|
|
1288
|
+
if (this.activeJobs.has(jobId))
|
|
1289
|
+
return;
|
|
1290
|
+
if (this.draining || this.loadingModel || this.activeJobs.size >= this.capacity()) {
|
|
1291
|
+
this.tunnel?.send({ type: 'error', jobId, code: 'host_busy', message: 'Agent draining, loading, or at capacity; retry on another host' });
|
|
1292
|
+
return;
|
|
1293
|
+
}
|
|
1294
|
+
// A buyer never waits behind the agent's own self-check.
|
|
1295
|
+
this.selfCheckAbort?.abort();
|
|
1296
|
+
const controller = new AbortController();
|
|
1297
|
+
const active = {
|
|
1298
|
+
jobId,
|
|
1299
|
+
controller,
|
|
1300
|
+
startedAt: Date.now(),
|
|
1301
|
+
firstTokenAt: null,
|
|
1302
|
+
chunks: 0,
|
|
1303
|
+
text: '',
|
|
1304
|
+
publicJob: !this.dashboard.current.job || this.dashboard.current.job.kind === 'public',
|
|
1305
|
+
requestId: this.dashboard.current.job?.id || null,
|
|
1306
|
+
};
|
|
1307
|
+
this.activeJobs.set(jobId, active);
|
|
1308
|
+
const wasIdle = this.activeJobs.size === 1;
|
|
1309
|
+
const perToken = this.earnsPerToken();
|
|
1310
|
+
try {
|
|
1311
|
+
if (wasIdle) {
|
|
1312
|
+
await this.sleepGuard.acquire(this.options.publicMode ? 'public token serving' : 'dedicated booking');
|
|
1313
|
+
}
|
|
1314
|
+
controller.signal.throwIfAborted();
|
|
1315
|
+
if (this.stopped)
|
|
1316
|
+
return;
|
|
1317
|
+
this.syncDashboard({ status: 'working', wakeLock: this.sleepGuard.status().active, message: null });
|
|
1318
|
+
const onChunk = (text, positions) => {
|
|
1319
|
+
if (this.stopped || controller.signal.aborted)
|
|
1320
|
+
return;
|
|
1321
|
+
if (active.firstTokenAt == null)
|
|
1322
|
+
active.firstTokenAt = Date.now();
|
|
1323
|
+
active.text += text;
|
|
1324
|
+
active.chunks++;
|
|
1325
|
+
const frame = positions.length ? { type: 'chunk', jobId, text, lp: positions } : { type: 'chunk', jobId, text };
|
|
1326
|
+
// A chunk lost to a tunnel drop would leave a hole in the buyer's text, so stop the job instead.
|
|
1327
|
+
if (!this.tunnel?.send(frame))
|
|
1328
|
+
controller.abort(new Error('Tunnel dropped during the stream'));
|
|
1329
|
+
};
|
|
1330
|
+
let result;
|
|
1331
|
+
if (this.options.simulation) {
|
|
1332
|
+
result = await simulateChat({
|
|
1333
|
+
jobId,
|
|
1334
|
+
messages: message.messages,
|
|
1335
|
+
maxTokens: Number(message.maxTokens || 256),
|
|
1336
|
+
temperature: Number(message.temperature ?? 0.7),
|
|
1337
|
+
model: this.model?.modelId || 'sublera-simulator',
|
|
1338
|
+
}, {
|
|
1339
|
+
signal: controller.signal,
|
|
1340
|
+
tokPerSec: this.options.tokPerSec || 24,
|
|
1341
|
+
onChunk,
|
|
1342
|
+
});
|
|
1343
|
+
}
|
|
1344
|
+
else {
|
|
1345
|
+
if (!this.runner)
|
|
1346
|
+
throw new Error('No model is loaded on this host');
|
|
1347
|
+
result = await streamChat(this.runner.url, {
|
|
1348
|
+
jobId,
|
|
1349
|
+
messages: message.messages,
|
|
1350
|
+
maxTokens: Number(message.maxTokens || 256),
|
|
1351
|
+
temperature: Number(message.temperature ?? 0.7),
|
|
1352
|
+
model: this.model?.modelId || 'local',
|
|
1353
|
+
}, {
|
|
1354
|
+
signal: controller.signal,
|
|
1355
|
+
onChunk,
|
|
1356
|
+
});
|
|
1357
|
+
}
|
|
1358
|
+
controller.signal.throwIfAborted();
|
|
1359
|
+
const elapsed = Math.max(1, Date.now() - active.startedAt);
|
|
1360
|
+
this.lastTtft = this.options.simulation ? 300 : (active.firstTokenAt ? active.firstTokenAt - active.startedAt : null);
|
|
1361
|
+
this.lastSpeed = this.options.simulation
|
|
1362
|
+
? (this.options.tokPerSec || 24)
|
|
1363
|
+
: (result.completionTokens > 0 ? result.completionTokens / (elapsed / 1000) : this.lastSpeed);
|
|
1364
|
+
this.tunnel?.send({
|
|
1365
|
+
type: 'done',
|
|
1366
|
+
jobId,
|
|
1367
|
+
finishReason: result.finishReason,
|
|
1368
|
+
promptTokens: result.promptTokens,
|
|
1369
|
+
completionTokens: result.completionTokens,
|
|
1370
|
+
});
|
|
1371
|
+
if (perToken)
|
|
1372
|
+
this.tokenEarnedNanos += this.estimateEarned(result.completionTokens);
|
|
1373
|
+
this.syncDashboard({ liveTokPerSec: this.lastSpeed, lastHeartbeatAt: Date.now() });
|
|
1374
|
+
}
|
|
1375
|
+
catch (error) {
|
|
1376
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
1377
|
+
if (!this.stopped)
|
|
1378
|
+
this.tunnel?.send({ type: 'error', jobId, code: 'infer_failed', message: msg });
|
|
1379
|
+
this.logger.warn('Inference failed', { error: msg });
|
|
1380
|
+
}
|
|
1381
|
+
finally {
|
|
1382
|
+
this.activeJobs.delete(jobId);
|
|
1383
|
+
if (this.activeJobs.size === 0) {
|
|
1384
|
+
await this.releaseWakeLockIfIdle();
|
|
1385
|
+
if (this.stopped)
|
|
1386
|
+
return;
|
|
1387
|
+
this.syncDashboard({
|
|
1388
|
+
status: this.bookings.size > 0 ? 'working' : 'online',
|
|
1389
|
+
wakeLock: this.sleepGuard.status().active,
|
|
1390
|
+
});
|
|
1391
|
+
}
|
|
1392
|
+
if (this.options.once && this.dashboard.current.job?.kind !== 'request')
|
|
1393
|
+
this.emit('once');
|
|
1394
|
+
}
|
|
1395
|
+
}
|
|
1396
|
+
/**
|
|
1397
|
+
* Whether the dashboard counts a job that starts now as per-token income: only while no paid
|
|
1398
|
+
* booking (speed seat or full GPU) is active. A renter's tokens are paid by the hour, and in
|
|
1399
|
+
* public mode per-token buyers are served beside speed seats, but an `infer` frame does not say
|
|
1400
|
+
* whose job it is. That is by design: the server must not reveal which jobs are hidden model
|
|
1401
|
+
* checks. So while rentals run, per-token earnings are left out of the session total; the
|
|
1402
|
+
* wallet on the website is exact.
|
|
1403
|
+
*/
|
|
1404
|
+
earnsPerToken() {
|
|
1405
|
+
for (const booking of this.bookings.values())
|
|
1406
|
+
if (booking.guaranteedNanos)
|
|
1407
|
+
return false;
|
|
1408
|
+
return true;
|
|
1409
|
+
}
|
|
1410
|
+
estimateEarned(completionTokens) {
|
|
1411
|
+
return Math.round((completionTokens / 1_000_000) * this.priceOutUsd() * 1e9);
|
|
1412
|
+
}
|
|
1413
|
+
/** The loaded model, when a load frame names it: the same SHA-256, or the same HuggingFace link when no hash is given. */
|
|
1414
|
+
loadedModelFor(load) {
|
|
1415
|
+
const model = this.model;
|
|
1416
|
+
if (!model)
|
|
1417
|
+
return null;
|
|
1418
|
+
if (!this.options.simulation && (!this.runner || this.runnerModelPath !== model.path))
|
|
1419
|
+
return null;
|
|
1420
|
+
const sha = typeof load.sha256 === 'string' ? load.sha256.trim().toLowerCase() : '';
|
|
1421
|
+
if (sha)
|
|
1422
|
+
return model.sha256?.toLowerCase() === sha ? model : null;
|
|
1423
|
+
try {
|
|
1424
|
+
return checkModelUrl(String(load.url || '')) === checkModelUrl(model.sourceUrl) ? model : null;
|
|
1425
|
+
}
|
|
1426
|
+
catch {
|
|
1427
|
+
return null;
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
/** The slots llama-server should run with after a load that asks for `loadSlots`. */
|
|
1431
|
+
slotsAfterLoad(loadSlots) {
|
|
1432
|
+
if (this.options.publicMode)
|
|
1433
|
+
return this.runnerSlots();
|
|
1434
|
+
return Math.max(1, this.options.slots ?? 1, loadSlots);
|
|
1435
|
+
}
|
|
1436
|
+
async handleLoad(load) {
|
|
1437
|
+
const requestId = String(load.requestId || '');
|
|
1438
|
+
if (this.stopped || this.draining || this.loadInFlight) {
|
|
1439
|
+
this.tunnel?.send({ type: 'load_failed', requestId, message: 'Host is busy; retry after current work completes' });
|
|
1440
|
+
return;
|
|
1441
|
+
}
|
|
1442
|
+
// Another request is loaded and in its model check (or a benchmark runs): it is not a booking
|
|
1443
|
+
// yet, but replacing it, or restarting llama-server under it, would break that check.
|
|
1444
|
+
const checking = this.checkingRequest();
|
|
1445
|
+
if (checking && checking.requestId !== requestId) {
|
|
1446
|
+
const what = checking.requestId.startsWith('bench_')
|
|
1447
|
+
? 'a benchmark is running on the loaded model'
|
|
1448
|
+
: `request ${checking.requestId} is loaded and waiting for its model check`;
|
|
1449
|
+
this.tunnel?.send({ type: 'load_failed', requestId, message: `Host is busy: it is serving other work (${what}). Retry after current work completes.` });
|
|
1450
|
+
return;
|
|
1451
|
+
}
|
|
1452
|
+
const loadSlots = Math.max(1, Math.min(16, Math.round(Number(load.slots) || 1)));
|
|
1453
|
+
const context = Math.max(Number(load.context || this.options.context || 8192), 2048);
|
|
1454
|
+
// The model this request names may already be running for other renters or public buyers.
|
|
1455
|
+
// Then the request is served on it as it is: no download, no VRAM check, no restart.
|
|
1456
|
+
const loaded = this.loadedModelFor(load);
|
|
1457
|
+
const plan = loaded ? this.runnerPlan(loaded.path, context, this.slotsAfterLoad(loadSlots), requestId) : null;
|
|
1458
|
+
const reuse = loaded && plan?.action === 'reuse' ? loaded : null;
|
|
1459
|
+
if (!reuse && this.runnerBusy(requestId)) {
|
|
1460
|
+
// Never stop llama-server under other bookings, public buyers, or a running job.
|
|
1461
|
+
const reason = plan?.action === 'refuse' && plan.note ? plan.note : 'it is serving other work on its loaded model, so it cannot load another model now';
|
|
1462
|
+
this.tunnel?.send({ type: 'load_failed', requestId, message: `Host is busy: ${reason}. Retry after current work completes.` });
|
|
1463
|
+
return;
|
|
1464
|
+
}
|
|
1465
|
+
this.loadInFlight = true;
|
|
1466
|
+
if (!reuse) {
|
|
1467
|
+
this.loadingModel = true;
|
|
1468
|
+
this.bookingSlots = loadSlots;
|
|
1469
|
+
}
|
|
1470
|
+
this.pending = { requestId, model: printable(String(load.file || load.url || '')) || null, verified: false, loaded: false, slots: loadSlots, context, holdsWake: true, seenAt: Date.now() };
|
|
1471
|
+
const loadAbort = { requestId, controller: new AbortController() };
|
|
1472
|
+
this.loadAbort = loadAbort;
|
|
1473
|
+
try {
|
|
1474
|
+
await this.sleepGuard.acquire(reuse ? 'serving a booked model' : 'downloading and loading a booked model');
|
|
1475
|
+
loadAbort.controller.signal.throwIfAborted();
|
|
1476
|
+
if (this.stopped || this.draining)
|
|
1477
|
+
throw new Error('Agent stopped or draining');
|
|
1478
|
+
this.syncDashboard({
|
|
1479
|
+
status: 'working',
|
|
1480
|
+
wakeLock: this.sleepGuard.status().active,
|
|
1481
|
+
message: requestId.startsWith('bench_') ? 'Loading a model for the benchmark' : reuse ? 'This model is already loaded; reusing it' : 'Pulling the requested GGUF',
|
|
1482
|
+
});
|
|
1483
|
+
let model;
|
|
1484
|
+
if (reuse) {
|
|
1485
|
+
// The wake-lock wait may have let another load or a stop change the engine.
|
|
1486
|
+
const now = this.runnerPlan(reuse.path, context, this.slotsAfterLoad(loadSlots), requestId);
|
|
1487
|
+
if (this.loadedModelFor(load) !== reuse || now.action !== 'reuse') {
|
|
1488
|
+
throw new Error('The loaded model changed while this request waited; retry');
|
|
1489
|
+
}
|
|
1490
|
+
if (now.note)
|
|
1491
|
+
this.logger.warn(now.note);
|
|
1492
|
+
this.tunnel?.send({ type: 'progress', requestId, note: 'Model already loaded; reusing the running engine' });
|
|
1493
|
+
model = reuse;
|
|
1494
|
+
}
|
|
1495
|
+
else {
|
|
1496
|
+
this.tunnel?.send({ type: 'progress', requestId, note: 'Starting download' });
|
|
1497
|
+
// The server may only point at a HuggingFace download, never at a file or address on this machine.
|
|
1498
|
+
const url = checkModelUrl(String(load.url || ''));
|
|
1499
|
+
model = await this.loadModel(url, {
|
|
1500
|
+
context,
|
|
1501
|
+
requestId,
|
|
1502
|
+
signal: loadAbort.controller.signal,
|
|
1503
|
+
expectedSha: load.sha256,
|
|
1504
|
+
expectedLayers: load.layers,
|
|
1505
|
+
file: load.file,
|
|
1506
|
+
});
|
|
1507
|
+
}
|
|
1508
|
+
loadAbort.controller.signal.throwIfAborted();
|
|
1509
|
+
if (this.stopped || this.draining)
|
|
1510
|
+
throw new Error('Agent stopped or draining');
|
|
1511
|
+
this.model = model;
|
|
1512
|
+
this.loadingModel = false;
|
|
1513
|
+
const servedContext = this.runnerContext ?? context;
|
|
1514
|
+
const pending = this.pending?.requestId === requestId ? this.pending : null;
|
|
1515
|
+
if (pending) {
|
|
1516
|
+
Object.assign(pending, { model: model.file, loaded: true, slots: this.capacity(), context: servedContext, seenAt: Date.now() });
|
|
1517
|
+
}
|
|
1518
|
+
this.tunnel?.send({
|
|
1519
|
+
type: 'loaded',
|
|
1520
|
+
requestId,
|
|
1521
|
+
sha256: model.sha256,
|
|
1522
|
+
file: model.file,
|
|
1523
|
+
layers: model.layers,
|
|
1524
|
+
context: servedContext,
|
|
1525
|
+
llamaUp: this.options.simulation || this.runner?.status === 'ready',
|
|
1526
|
+
});
|
|
1527
|
+
this.syncDashboard({
|
|
1528
|
+
llamaStatus: this.runner?.status || 'simulation',
|
|
1529
|
+
message: reuse ? 'Model already loaded. Waiting for the server model check.'
|
|
1530
|
+
: this.options.simulation ? 'Simulation loaded. Waiting for the server check.'
|
|
1531
|
+
: model.verified ? 'File hash matched. Waiting for the server model check.'
|
|
1532
|
+
: 'Model loaded without a reference hash. Waiting for the server model check.',
|
|
1533
|
+
});
|
|
1534
|
+
}
|
|
1535
|
+
catch (error) {
|
|
1536
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1537
|
+
this.loadingModel = false;
|
|
1538
|
+
if (this.pending?.requestId === requestId)
|
|
1539
|
+
this.pending = null;
|
|
1540
|
+
await this.releaseWakeLockIfIdle();
|
|
1541
|
+
if (this.stopped)
|
|
1542
|
+
return;
|
|
1543
|
+
this.tunnel?.send({ type: 'load_failed', requestId, message });
|
|
1544
|
+
this.logger.error('Request load failed', { error: message });
|
|
1545
|
+
this.syncDashboard({ status: this.bookings.size ? 'working' : 'online', message: `Load failed: ${message}` });
|
|
1546
|
+
}
|
|
1547
|
+
finally {
|
|
1548
|
+
this.loadInFlight = false;
|
|
1549
|
+
if (this.loadAbort?.requestId === requestId)
|
|
1550
|
+
this.loadAbort = null;
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
async refreshMe() {
|
|
1554
|
+
// Bookings confirmed after this moment are newer than the answer and are never dropped by it.
|
|
1555
|
+
const askedAt = Date.now();
|
|
1556
|
+
try {
|
|
1557
|
+
const me = await fetchJson(apiUrl(this.options.config.server, '/api/agent/me'), {
|
|
1558
|
+
token: this.options.config.token,
|
|
1559
|
+
timeoutMs: 10_000,
|
|
1560
|
+
});
|
|
1561
|
+
if (this.stopped)
|
|
1562
|
+
return;
|
|
1563
|
+
this.noteServerClock(me);
|
|
1564
|
+
const endpoint = await this.endpointFor(this.options.config);
|
|
1565
|
+
if (this.stopped)
|
|
1566
|
+
return;
|
|
1567
|
+
this.me = me;
|
|
1568
|
+
if (this.meInitialWallet == null)
|
|
1569
|
+
this.meInitialWallet = me.wallet.availableNanos;
|
|
1570
|
+
const host = me.host;
|
|
1571
|
+
const ended = this.reconcileBookings(me, askedAt);
|
|
1572
|
+
this.syncDashboard({
|
|
1573
|
+
username: me.user.username,
|
|
1574
|
+
endpoint,
|
|
1575
|
+
reliability: host?.uptimePct ?? null,
|
|
1576
|
+
held: host?.status === 'held',
|
|
1577
|
+
walletNanos: me.wallet.availableNanos + me.wallet.pendingNanos,
|
|
1578
|
+
liveTokPerSec: this.lastSpeed || this.dashboard.current.liveTokPerSec,
|
|
1579
|
+
lastHeartbeatAt: this.heartbeatAt,
|
|
1580
|
+
...(ended.length ? {
|
|
1581
|
+
status: this.bookings.size || this.activeJobs.size ? 'working' : 'online',
|
|
1582
|
+
message: ended.length === 1 ? `Booking ${ended[0]} ended` : `${ended.length} bookings ended`,
|
|
1583
|
+
} : {}),
|
|
1584
|
+
});
|
|
1585
|
+
}
|
|
1586
|
+
catch (error) {
|
|
1587
|
+
if (error instanceof HttpError && (error.status === 404 || error.status === 405)) {
|
|
1588
|
+
this.logger.debug('Server does not expose /api/agent/me; dashboard will use tunnel state only.');
|
|
1589
|
+
}
|
|
1590
|
+
else {
|
|
1591
|
+
this.logger.debug('Could not refresh host status', { error: error instanceof Error ? error.message : String(error) });
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
}
|
|
1595
|
+
/**
|
|
1596
|
+
* Makes the bookings match the server's list of live requests: adds booked ones this agent
|
|
1597
|
+
* missed (never one that already ended here or whose end has passed on the server's clock),
|
|
1598
|
+
* updates timers, and drops bookings the server no longer lists. Also keeps the ids of every
|
|
1599
|
+
* live request for servingOthers(). Returns the dropped ids.
|
|
1600
|
+
*/
|
|
1601
|
+
reconcileBookings(me, askedAt) {
|
|
1602
|
+
// activeRequestsOf drops rows with a malformed id and turns broken numbers into null.
|
|
1603
|
+
const listed = activeRequestsOf(me);
|
|
1604
|
+
const ids = new Set(listed.map((row) => row.id));
|
|
1605
|
+
// Whether a listed booking is over is judged by the server's clock, not this machine's.
|
|
1606
|
+
const now = this.serverTime();
|
|
1607
|
+
const over = (row) => row.status === 'active' && row.timerEndsAt != null && row.timerEndsAt <= now;
|
|
1608
|
+
this.liveRequestIds = new Set(listed
|
|
1609
|
+
.filter((row) => LIVE_STATUSES.has(row.status) && !this.endedIds.has(row.id) && !over(row))
|
|
1610
|
+
.map((row) => row.id));
|
|
1611
|
+
const newest = isRequestId(me.activeRequest?.id) ? me.activeRequest.id : null;
|
|
1612
|
+
let changed = newest !== this.serverNewestId && this.bookings.size > 0;
|
|
1613
|
+
this.serverNewestId = newest;
|
|
1614
|
+
let added = false;
|
|
1615
|
+
for (const row of listed) {
|
|
1616
|
+
if (row.status !== 'active' && row.status !== 'awaiting_confirm')
|
|
1617
|
+
continue;
|
|
1618
|
+
// A booking that ended here stays ended, whatever an answer built before the end still says.
|
|
1619
|
+
if (this.endedIds.has(row.id) || over(row))
|
|
1620
|
+
continue;
|
|
1621
|
+
const existing = this.bookings.get(row.id);
|
|
1622
|
+
// A running booking always has an end time; a row without a valid one is not added.
|
|
1623
|
+
if (!existing && row.status === 'active' && !row.timerEndsAt)
|
|
1624
|
+
continue;
|
|
1625
|
+
if (!existing)
|
|
1626
|
+
added = true;
|
|
1627
|
+
const hours = row.hours || existing?.hours || null;
|
|
1628
|
+
const price = row.pricePerHourNanos || existing?.pricePerHourNanos || null;
|
|
1629
|
+
const tokS = row.tokS === undefined ? undefined : row.tokS && row.tokS > 0 ? row.tokS : null;
|
|
1630
|
+
this.upsertBooking(row.id, {
|
|
1631
|
+
model: existing?.model ?? (row.file ? printable(row.file) : undefined),
|
|
1632
|
+
verified: Boolean(row.timerStartedAt) || existing?.verified || (this.pending?.requestId === row.id && this.pending.verified),
|
|
1633
|
+
timerStartedAt: row.timerStartedAt || undefined,
|
|
1634
|
+
timerEndsAt: row.timerEndsAt || undefined,
|
|
1635
|
+
hours: hours ?? undefined,
|
|
1636
|
+
pricePerHourNanos: price ?? undefined,
|
|
1637
|
+
minTokS: row.minTokS === undefined ? undefined : row.minTokS || null,
|
|
1638
|
+
guaranteedNanos: hours && price ? Math.round(hours * price) : undefined,
|
|
1639
|
+
shared: row.shared == null ? (tokS != null ? true : undefined) : Boolean(row.shared),
|
|
1640
|
+
tokS,
|
|
1641
|
+
});
|
|
1642
|
+
changed = true;
|
|
1643
|
+
}
|
|
1644
|
+
const ended = [];
|
|
1645
|
+
for (const booking of [...this.bookings.values()]) {
|
|
1646
|
+
if (ids.has(booking.requestId) || booking.seenAt >= askedAt)
|
|
1647
|
+
continue;
|
|
1648
|
+
this.removeBooking(booking.requestId);
|
|
1649
|
+
ended.push(booking.requestId);
|
|
1650
|
+
changed = true;
|
|
1651
|
+
}
|
|
1652
|
+
if (ended.length)
|
|
1653
|
+
this.logger.info('Bookings ended on the server', { requestIds: ended.join(',') });
|
|
1654
|
+
// A loaded request the server stopped listing no longer keeps the machine awake.
|
|
1655
|
+
const pending = this.pending;
|
|
1656
|
+
let released = ended.length > 0;
|
|
1657
|
+
let wake = added;
|
|
1658
|
+
if (pending && !this.loadInFlight) {
|
|
1659
|
+
if (ids.has(pending.requestId)) {
|
|
1660
|
+
// Listed again after a gap: it keeps the machine awake again.
|
|
1661
|
+
if (!pending.holdsWake)
|
|
1662
|
+
wake = true;
|
|
1663
|
+
pending.holdsWake = true;
|
|
1664
|
+
}
|
|
1665
|
+
else if (pending.holdsWake && this.activeJobs.size === 0 && pending.seenAt < askedAt) {
|
|
1666
|
+
pending.holdsWake = false;
|
|
1667
|
+
released = true;
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
if (changed)
|
|
1671
|
+
this.persistBookings();
|
|
1672
|
+
if (wake)
|
|
1673
|
+
this.holdWakeLock();
|
|
1674
|
+
if (released)
|
|
1675
|
+
void this.releaseWakeLockIfIdle();
|
|
1676
|
+
return ended;
|
|
1677
|
+
}
|
|
1678
|
+
setupKeyboard() {
|
|
1679
|
+
if (!process.stdin.isTTY || !process.stdout.isTTY || this.keyHandler)
|
|
1680
|
+
return;
|
|
1681
|
+
const handler = (data) => { void this.handleKey(data.toString()); };
|
|
1682
|
+
this.keyHandler = handler;
|
|
1683
|
+
process.stdin.setRawMode(true);
|
|
1684
|
+
process.stdin.resume();
|
|
1685
|
+
process.stdin.on('data', handler);
|
|
1686
|
+
this.dashboard.update({ keyHint: 'W host page · C copy endpoint · L logs · ? help' });
|
|
1687
|
+
}
|
|
1688
|
+
async handleKey(key) {
|
|
1689
|
+
if (key === '\u0003') {
|
|
1690
|
+
await this.stop();
|
|
1691
|
+
return;
|
|
1692
|
+
}
|
|
1693
|
+
if (key === 'w' || key === 'W') {
|
|
1694
|
+
const url = this.hostPageUrl();
|
|
1695
|
+
this.openExternal(url);
|
|
1696
|
+
this.dashboard.update({ message: `Host page: ${url}` });
|
|
1697
|
+
return;
|
|
1698
|
+
}
|
|
1699
|
+
if (key === 'c' || key === 'C') {
|
|
1700
|
+
const endpoint = this.dashboard.current.endpoint || '';
|
|
1701
|
+
const copied = await this.copyText(endpoint);
|
|
1702
|
+
this.dashboard.update({ message: copied ? `Copied ${endpoint}` : `Endpoint: ${endpoint}` });
|
|
1703
|
+
return;
|
|
1704
|
+
}
|
|
1705
|
+
if (key === 'l' || key === 'L') {
|
|
1706
|
+
const showLogs = !this.dashboard.current.showLogs;
|
|
1707
|
+
this.dashboard.update({ showLogs, logTail: this.logLines.slice(-5) });
|
|
1708
|
+
return;
|
|
1709
|
+
}
|
|
1710
|
+
if (key === '?') {
|
|
1711
|
+
this.dashboard.update({ message: 'W host page · C copy endpoint · L logs · Ctrl+C stop' });
|
|
1712
|
+
}
|
|
1713
|
+
}
|
|
1714
|
+
/** The website page with this host's earnings, reliability, and current request. */
|
|
1715
|
+
hostPageUrl() {
|
|
1716
|
+
return `${this.options.config.server.replace(/\/+$/, '')}/host`;
|
|
1717
|
+
}
|
|
1718
|
+
openExternal(url) {
|
|
1719
|
+
openUrl(url, this.options.config.server);
|
|
1720
|
+
}
|
|
1721
|
+
async copyText(text) {
|
|
1722
|
+
if (!text)
|
|
1723
|
+
return false;
|
|
1724
|
+
try {
|
|
1725
|
+
const command = process.platform === 'win32' ? 'clip' : process.platform === 'darwin' ? 'pbcopy' : 'xclip';
|
|
1726
|
+
const args = process.platform === 'linux' ? ['-selection', 'clipboard'] : [];
|
|
1727
|
+
const result = await execFile(command, args, { input: text, timeoutMs: 5_000 });
|
|
1728
|
+
return result.code === 0;
|
|
1729
|
+
}
|
|
1730
|
+
catch {
|
|
1731
|
+
return false;
|
|
1732
|
+
}
|
|
1733
|
+
}
|
|
1734
|
+
cleanupKeyboard() {
|
|
1735
|
+
if (!this.keyHandler)
|
|
1736
|
+
return;
|
|
1737
|
+
process.stdin.off('data', this.keyHandler);
|
|
1738
|
+
this.keyHandler = null;
|
|
1739
|
+
try {
|
|
1740
|
+
process.stdin.setRawMode(false);
|
|
1741
|
+
}
|
|
1742
|
+
catch { /* not a TTY */ }
|
|
1743
|
+
process.stdin.pause();
|
|
1744
|
+
}
|
|
1745
|
+
startGpuPolling() {
|
|
1746
|
+
if (this.options.simulation)
|
|
1747
|
+
return;
|
|
1748
|
+
this.gpuTimer = setInterval(() => {
|
|
1749
|
+
void (async () => {
|
|
1750
|
+
try {
|
|
1751
|
+
this.gpu = applyGpuOverride(await this.gpuMonitor.scan(true), this.options.gpuOverride);
|
|
1752
|
+
this.updateGpuDashboard();
|
|
1753
|
+
const ownPid = this.runner?.pid;
|
|
1754
|
+
const others = (this.gpu.primary?.processes || []).filter((proc) => proc.pid !== ownPid && !/llama-server|llama\.exe|server\.exe/i.test(proc.name) && proc.usedMb >= 256);
|
|
1755
|
+
if (this.activeJobs.size > 0 && others.length) {
|
|
1756
|
+
const note = `Another process is using GPU memory: ${others.map((p) => `${p.name.split(/[\\/]/).pop()} ${p.usedMb}MB`).join(', ')}`;
|
|
1757
|
+
this.logger.warn(note);
|
|
1758
|
+
this.dashboard.update({ message: note });
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
catch {
|
|
1762
|
+
/* keep the heartbeat alive even if nvidia-smi hiccups */
|
|
1763
|
+
}
|
|
1764
|
+
})();
|
|
1765
|
+
}, 5_000);
|
|
1766
|
+
}
|
|
1767
|
+
async releaseWakeLockIfIdle() {
|
|
1768
|
+
if (this.bookingWake || this.activeJobs.size > 0)
|
|
1769
|
+
return;
|
|
1770
|
+
await this.sleepGuard.stop();
|
|
1771
|
+
if (!this.stopped)
|
|
1772
|
+
this.dashboard.update({ wakeLock: false });
|
|
1773
|
+
}
|
|
1774
|
+
updateGpuDashboard() {
|
|
1775
|
+
const gpu = this.gpu.primary;
|
|
1776
|
+
this.dashboard.update({
|
|
1777
|
+
gpuName: gpu?.name || this.options.gpuOverride?.name || null,
|
|
1778
|
+
totalVramMb: gpu?.totalVramMb || this.options.gpuOverride?.vramMb || 0,
|
|
1779
|
+
freeVramMb: gpu?.freeVramMb || 0,
|
|
1780
|
+
gpuTempC: gpu?.temperatureC ?? null,
|
|
1781
|
+
gpuPowerW: gpu?.powerW ?? null,
|
|
1782
|
+
gpuUtilPct: gpu?.utilizationPct ?? null,
|
|
1783
|
+
});
|
|
1784
|
+
}
|
|
1785
|
+
fail(error, status = 'auth-failed') {
|
|
1786
|
+
this.logger.error(error.message);
|
|
1787
|
+
this.dashboard.update({ status, message: error.message });
|
|
1788
|
+
this.emit('fatal', error);
|
|
1789
|
+
}
|
|
1790
|
+
stop(preserveState = false) {
|
|
1791
|
+
if (!this.stopPromise)
|
|
1792
|
+
this.stopPromise = this.stopOwned(preserveState);
|
|
1793
|
+
return this.stopPromise;
|
|
1794
|
+
}
|
|
1795
|
+
async stopOwned(preserveState) {
|
|
1796
|
+
this.stopped = true;
|
|
1797
|
+
this.online = false;
|
|
1798
|
+
for (const waiter of this.onlineWaiters) {
|
|
1799
|
+
clearTimeout(waiter.timer);
|
|
1800
|
+
waiter.reject(new Error('Agent stopped'));
|
|
1801
|
+
}
|
|
1802
|
+
this.onlineWaiters = [];
|
|
1803
|
+
if (this.meTimer)
|
|
1804
|
+
clearInterval(this.meTimer);
|
|
1805
|
+
if (this.gpuTimer)
|
|
1806
|
+
clearInterval(this.gpuTimer);
|
|
1807
|
+
if (this.bookingTimer)
|
|
1808
|
+
clearInterval(this.bookingTimer);
|
|
1809
|
+
this.clearBookingTimers();
|
|
1810
|
+
this.meTimer = null;
|
|
1811
|
+
this.gpuTimer = null;
|
|
1812
|
+
this.bookingTimer = null;
|
|
1813
|
+
// In memory only: with preserveState the bookings stay in state.json for the next start.
|
|
1814
|
+
this.bookings.clear();
|
|
1815
|
+
this.pending = null;
|
|
1816
|
+
this.loadingModel = false;
|
|
1817
|
+
// Ctrl+C clears saved state, also while a restore is still working on it; the restore then stops early.
|
|
1818
|
+
if (!preserveState)
|
|
1819
|
+
this.clearPersistedBooking(this.restoring);
|
|
1820
|
+
this.selfCheckAbort?.abort();
|
|
1821
|
+
this.loadAbort?.controller.abort(new Error('Agent stopping'));
|
|
1822
|
+
for (const job of this.activeJobs.values())
|
|
1823
|
+
job.controller.abort(new Error('Agent stopping'));
|
|
1824
|
+
this.activeJobs.clear();
|
|
1825
|
+
this.startup.abort(new Error('Agent stopping'));
|
|
1826
|
+
try {
|
|
1827
|
+
await this.releaseResources();
|
|
1828
|
+
}
|
|
1829
|
+
finally {
|
|
1830
|
+
this.emit('stop');
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
/** Frees the wake lock, tunnel, llama-server and terminal. run() calls it again when startup finishes after stop(). */
|
|
1834
|
+
async releaseResources() {
|
|
1835
|
+
for (const timer of [this.meTimer, this.gpuTimer, this.bookingTimer])
|
|
1836
|
+
if (timer)
|
|
1837
|
+
clearInterval(timer);
|
|
1838
|
+
this.clearBookingTimers();
|
|
1839
|
+
this.meTimer = null;
|
|
1840
|
+
this.gpuTimer = null;
|
|
1841
|
+
this.bookingTimer = null;
|
|
1842
|
+
const errors = [];
|
|
1843
|
+
for (const cleanup of [
|
|
1844
|
+
() => this.cleanupKeyboard(),
|
|
1845
|
+
() => this.sleepGuard.stop(),
|
|
1846
|
+
() => this.tunnel?.stop(),
|
|
1847
|
+
() => this.runner?.stop(),
|
|
1848
|
+
() => {
|
|
1849
|
+
this.runner = null;
|
|
1850
|
+
this.runnerModelPath = null;
|
|
1851
|
+
this.runnerContext = null;
|
|
1852
|
+
this.runnerSlotCount = null;
|
|
1853
|
+
this.dashboard.stop();
|
|
1854
|
+
},
|
|
1855
|
+
]) {
|
|
1856
|
+
try {
|
|
1857
|
+
await cleanup();
|
|
1858
|
+
}
|
|
1859
|
+
catch (error) {
|
|
1860
|
+
errors.push(error);
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
if (errors.length)
|
|
1864
|
+
throw errors[0];
|
|
1865
|
+
}
|
|
1866
|
+
}
|
|
1867
|
+
//# sourceMappingURL=agent.js.map
|