pi-harness-runtime 1.1.13 → 1.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -26
- package/footer-status.ts +49 -4
- package/harness/glm-quota-countdown.test.ts +323 -0
- package/harness/glm-quota-countdown.ts +615 -0
- package/harness/index.ts +12 -0
- package/package.json +1 -1
- package/packages/quota-manager/dist/tui-usage-monitor.d.ts.map +1 -1
- package/packages/quota-manager/dist/tui-usage-monitor.js +20 -4
- package/packages/quota-manager/dist/tui-usage-monitor.js.map +1 -1
- package/packages/quota-manager/src/tui-usage-monitor.ts +21 -4
|
@@ -0,0 +1,615 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GLM Quota Countdown — Robust countdown system for GLM quota exhaustion
|
|
3
|
+
*
|
|
4
|
+
* Features:
|
|
5
|
+
* - Stores reset time when 429 error is detected
|
|
6
|
+
* - Runs countdown timer with periodic status updates
|
|
7
|
+
* - Sends notifications at 15min, 5min, 1min intervals
|
|
8
|
+
* - Updates TUI footer with countdown every minute
|
|
9
|
+
* - Auto-triggers resume when reset time arrives
|
|
10
|
+
*
|
|
11
|
+
* Flow:
|
|
12
|
+
* 429 Error Detected
|
|
13
|
+
* → parseResetTime() extracts ISO timestamp from error message
|
|
14
|
+
* → startCountdown() begins monitoring
|
|
15
|
+
* → Every tick: updateFooterStatus() refreshes TUI
|
|
16
|
+
* → At intervals: sendNotification() alerts user
|
|
17
|
+
* → At reset: triggerAutoResume() continues the job
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import type { JobStateMachine } from "./job-state-machine.js";
|
|
21
|
+
import type { MirrorStore } from "../mirror.js";
|
|
22
|
+
import type { NotificationCenter } from "../packages/notification/dist/notification-center.js";
|
|
23
|
+
|
|
24
|
+
/** Notification intervals before reset (in seconds) */
|
|
25
|
+
const NOTIFICATION_INTERVALS = [
|
|
26
|
+
15 * 60, // 15 minutes
|
|
27
|
+
5 * 60, // 5 minutes
|
|
28
|
+
60, // 1 minute
|
|
29
|
+
] as const;
|
|
30
|
+
|
|
31
|
+
/** How often to update the TUI footer (in seconds) */
|
|
32
|
+
const FOOTER_UPDATE_INTERVAL_SECONDS = 60; // Every minute
|
|
33
|
+
|
|
34
|
+
/** Buffer before reset time to attempt resume (ms) */
|
|
35
|
+
const RESUME_BUFFER_MS = 10_000; // 10 seconds
|
|
36
|
+
|
|
37
|
+
/** Minimum delay before auto-resume (ms) */
|
|
38
|
+
const MIN_RESUME_DELAY_MS = 5_000; // 5 seconds
|
|
39
|
+
|
|
40
|
+
export interface GLMQuotaCountdownState {
|
|
41
|
+
/** Job ID this countdown is for */
|
|
42
|
+
jobId: string;
|
|
43
|
+
/** When the quota resets (ISO string) */
|
|
44
|
+
resetAt: string;
|
|
45
|
+
/** When the quota resets (epoch ms) */
|
|
46
|
+
resetAtEpoch: number;
|
|
47
|
+
/** When countdown started */
|
|
48
|
+
startedAt: string;
|
|
49
|
+
/** Total seconds until reset */
|
|
50
|
+
totalSeconds: number;
|
|
51
|
+
/** Last notification sent (type) */
|
|
52
|
+
lastNotificationSent?: "15min" | "5min" | "1min" | "reset";
|
|
53
|
+
/** Auto-resume scheduled */
|
|
54
|
+
autoResumeScheduled: boolean;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface CountdownTickEvent {
|
|
58
|
+
jobId: string;
|
|
59
|
+
remainingSeconds: number;
|
|
60
|
+
remainingFormatted: string;
|
|
61
|
+
nextNotificationIn?: number;
|
|
62
|
+
isResetTime: boolean;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
type CountdownCallback = (event: CountdownTickEvent) => void;
|
|
66
|
+
|
|
67
|
+
/** Regex patterns to extract reset time from GLM 429 errors */
|
|
68
|
+
const GLM_RESET_TIME_PATTERNS = [
|
|
69
|
+
// "Your limit will reset at 2026-08-25 01:47:16"
|
|
70
|
+
/reset at (\d{4}-\d{2}-\d{2}[T ]?\d{2}:\d{2}:\d{2})/i,
|
|
71
|
+
// "reset at 2026-08-25 01:47:16" (alternative format)
|
|
72
|
+
/reset\s+(?:at\s+)?(\d{4}-\d{2}-\d{2}[T ]?\d{2}:\d{2}:\d{2})/i,
|
|
73
|
+
// "will reset at 01:47:16" (date from context)
|
|
74
|
+
/will reset at (\d{2}:\d{2}:\d{2})/i,
|
|
75
|
+
];
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Parse reset time from GLM 429 error message
|
|
79
|
+
* Returns ISO string or null if not found
|
|
80
|
+
*/
|
|
81
|
+
export function parseGLMResetTime(errorMessage: string): string | null {
|
|
82
|
+
// Try full datetime patterns first
|
|
83
|
+
for (const pattern of GLM_RESET_TIME_PATTERNS.slice(0, 2)) {
|
|
84
|
+
const match = errorMessage.match(pattern);
|
|
85
|
+
if (match) {
|
|
86
|
+
const datetimeStr = match[1];
|
|
87
|
+
// Check if it's a full ISO datetime
|
|
88
|
+
if (datetimeStr.includes("-")) {
|
|
89
|
+
// Already has date, just ensure it's parseable
|
|
90
|
+
const date = new Date(datetimeStr.replace(" ", "T"));
|
|
91
|
+
if (!isNaN(date.getTime())) {
|
|
92
|
+
return date.toISOString();
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Try time-only pattern - need to determine the date
|
|
99
|
+
const timeMatch = errorMessage.match(/(\d{2}):(\d{2}):(\d{2})/);
|
|
100
|
+
if (timeMatch) {
|
|
101
|
+
const [, hour, minute, second] = timeMatch;
|
|
102
|
+
const now = new Date();
|
|
103
|
+
const resetTime = new Date(
|
|
104
|
+
now.getFullYear(),
|
|
105
|
+
now.getMonth(),
|
|
106
|
+
now.getDate(),
|
|
107
|
+
parseInt(hour, 10),
|
|
108
|
+
parseInt(minute, 10),
|
|
109
|
+
parseInt(second, 10),
|
|
110
|
+
0,
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
// If time has passed today, assume tomorrow
|
|
114
|
+
if (resetTime <= now) {
|
|
115
|
+
resetTime.setDate(resetTime.getDate() + 1);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
return resetTime.toISOString();
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Format seconds into human-readable countdown string
|
|
126
|
+
*/
|
|
127
|
+
export function formatCountdown(seconds: number): string {
|
|
128
|
+
if (seconds <= 0) return "RESET NOW";
|
|
129
|
+
|
|
130
|
+
const days = Math.floor(seconds / 86400);
|
|
131
|
+
const hours = Math.floor((seconds % 86400) / 3600);
|
|
132
|
+
const minutes = Math.floor((seconds % 3600) / 60);
|
|
133
|
+
const secs = seconds % 60;
|
|
134
|
+
|
|
135
|
+
const parts: string[] = [];
|
|
136
|
+
if (days > 0) parts.push(`${days}d`);
|
|
137
|
+
if (hours > 0 || days > 0) parts.push(`${hours}h`);
|
|
138
|
+
if (minutes > 0 || hours > 0 || days > 0) parts.push(`${minutes}m`);
|
|
139
|
+
parts.push(`${secs}s`);
|
|
140
|
+
|
|
141
|
+
return parts.join(" ");
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* GLM Quota Countdown Manager
|
|
146
|
+
*
|
|
147
|
+
* Manages countdown timers for GLM quota exhaustion, providing:
|
|
148
|
+
* - Real-time countdown display in TUI
|
|
149
|
+
* - Periodic notifications before reset
|
|
150
|
+
* - Auto-resume capability
|
|
151
|
+
*/
|
|
152
|
+
export class GLMQuotaCountdown {
|
|
153
|
+
private activeCountdowns: Map<string, GLMQuotaCountdownState> = new Map();
|
|
154
|
+
private timers: Map<string, NodeJS.Timeout> = new Map();
|
|
155
|
+
private tickCallbacks: Set<CountdownCallback> = new Set();
|
|
156
|
+
private notificationCenter: NotificationCenter | null = null;
|
|
157
|
+
private jobContext: { jobId: string; requirement: string } | null = null;
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Set notification center for sending alerts
|
|
161
|
+
*/
|
|
162
|
+
setNotificationCenter(center: NotificationCenter): void {
|
|
163
|
+
this.notificationCenter = center;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Set job context for notifications
|
|
168
|
+
*/
|
|
169
|
+
setJobContext(jobId: string, requirement: string): void {
|
|
170
|
+
this.jobContext = { jobId, requirement };
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Register a tick callback (e.g., for TUI footer updates)
|
|
175
|
+
*/
|
|
176
|
+
onTick(callback: CountdownCallback): () => void {
|
|
177
|
+
this.tickCallbacks.add(callback);
|
|
178
|
+
return () => this.tickCallbacks.delete(callback);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Check if there's an active countdown for a job
|
|
183
|
+
*/
|
|
184
|
+
hasCountdown(jobId: string): boolean {
|
|
185
|
+
return this.activeCountdowns.has(jobId);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Get current countdown state for a job
|
|
190
|
+
*/
|
|
191
|
+
getCountdown(jobId: string): GLMQuotaCountdownState | null {
|
|
192
|
+
return this.activeCountdowns.get(jobId) ?? null;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Get all active countdowns
|
|
197
|
+
*/
|
|
198
|
+
getAllCountdowns(): GLMQuotaCountdownState[] {
|
|
199
|
+
return Array.from(this.activeCountdowns.values());
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Start a countdown for a job when GLM quota is exhausted
|
|
204
|
+
*
|
|
205
|
+
* @param jobId - Job identifier
|
|
206
|
+
* @param resetAt - ISO timestamp when quota resets
|
|
207
|
+
* @param mirrorStore - MirrorStore for updating quota status
|
|
208
|
+
* @param machine - JobStateMachine for auto-resume
|
|
209
|
+
*/
|
|
210
|
+
async startCountdown(
|
|
211
|
+
jobId: string,
|
|
212
|
+
resetAt: string,
|
|
213
|
+
mirrorStore: MirrorStore,
|
|
214
|
+
machine: JobStateMachine,
|
|
215
|
+
): Promise<GLMQuotaCountdownState> {
|
|
216
|
+
// Cancel any existing countdown for this job
|
|
217
|
+
this.cancelCountdown(jobId);
|
|
218
|
+
|
|
219
|
+
const resetAtEpoch = new Date(resetAt).getTime();
|
|
220
|
+
const now = Date.now();
|
|
221
|
+
const totalSeconds = Math.max(0, Math.floor((resetAtEpoch - now) / 1000));
|
|
222
|
+
|
|
223
|
+
const state: GLMQuotaCountdownState = {
|
|
224
|
+
jobId,
|
|
225
|
+
resetAt,
|
|
226
|
+
resetAtEpoch,
|
|
227
|
+
startedAt: new Date().toISOString(),
|
|
228
|
+
totalSeconds,
|
|
229
|
+
autoResumeScheduled: false,
|
|
230
|
+
};
|
|
231
|
+
|
|
232
|
+
this.activeCountdowns.set(jobId, state);
|
|
233
|
+
|
|
234
|
+
// Update mirror store with reset time
|
|
235
|
+
this.updateMirrorWithResetTime(jobId, resetAt, resetAtEpoch, mirrorStore);
|
|
236
|
+
|
|
237
|
+
// Schedule the countdown timer
|
|
238
|
+
this.scheduleCountdown(jobId, state, mirrorStore, machine);
|
|
239
|
+
|
|
240
|
+
// Emit initial tick
|
|
241
|
+
this.emitTick({
|
|
242
|
+
jobId,
|
|
243
|
+
remainingSeconds: totalSeconds,
|
|
244
|
+
remainingFormatted: formatCountdown(totalSeconds),
|
|
245
|
+
isResetTime: totalSeconds <= 0,
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
console.log(
|
|
249
|
+
`[GLMQuotaCountdown] Started countdown for job ${jobId}: ${formatCountdown(totalSeconds)} until reset at ${resetAt}`,
|
|
250
|
+
);
|
|
251
|
+
|
|
252
|
+
return state;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Start countdown from a 429 error message
|
|
257
|
+
*/
|
|
258
|
+
async startFromError(
|
|
259
|
+
jobId: string,
|
|
260
|
+
errorMessage: string,
|
|
261
|
+
mirrorStore: MirrorStore,
|
|
262
|
+
machine: JobStateMachine,
|
|
263
|
+
): Promise<GLMQuotaCountdownState | null> {
|
|
264
|
+
const resetAt = parseGLMResetTime(errorMessage);
|
|
265
|
+
if (!resetAt) {
|
|
266
|
+
console.warn(
|
|
267
|
+
`[GLMQuotaCountdown] Could not parse reset time from error for job ${jobId}`,
|
|
268
|
+
);
|
|
269
|
+
return null;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
return this.startCountdown(jobId, resetAt, mirrorStore, machine);
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* Cancel countdown for a job
|
|
277
|
+
*/
|
|
278
|
+
cancelCountdown(jobId: string): void {
|
|
279
|
+
const existingTimer = this.timers.get(jobId);
|
|
280
|
+
if (existingTimer) {
|
|
281
|
+
clearTimeout(existingTimer);
|
|
282
|
+
this.timers.delete(jobId);
|
|
283
|
+
}
|
|
284
|
+
this.activeCountdowns.delete(jobId);
|
|
285
|
+
console.log(`[GLMQuotaCountdown] Cancelled countdown for job ${jobId}`);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Update mirror store with reset time info
|
|
290
|
+
*/
|
|
291
|
+
private updateMirrorWithResetTime(
|
|
292
|
+
jobId: string,
|
|
293
|
+
resetAt: string,
|
|
294
|
+
resetAtEpoch: number,
|
|
295
|
+
mirrorStore: MirrorStore,
|
|
296
|
+
): void {
|
|
297
|
+
try {
|
|
298
|
+
const record = mirrorStore.readProvider("glm") ?? {
|
|
299
|
+
synced_at: new Date().toISOString(),
|
|
300
|
+
provider: "glm",
|
|
301
|
+
source: "tui-signal" as const,
|
|
302
|
+
exhausted: true,
|
|
303
|
+
limitType: "tokens" as const,
|
|
304
|
+
resets_at: resetAt,
|
|
305
|
+
};
|
|
306
|
+
|
|
307
|
+
// Update with exhaustion info
|
|
308
|
+
const updated = {
|
|
309
|
+
...record,
|
|
310
|
+
synced_at: new Date().toISOString(),
|
|
311
|
+
provider: "glm",
|
|
312
|
+
source: (record.source || "tui-signal") as
|
|
313
|
+
| "scrape"
|
|
314
|
+
| "tui-signal"
|
|
315
|
+
| "manual",
|
|
316
|
+
exhausted: true,
|
|
317
|
+
limitType: record.limitType || ("tokens" as const),
|
|
318
|
+
h5_used_pct: 100,
|
|
319
|
+
h5_resets_at: resetAt,
|
|
320
|
+
h5_resets_at_epoch: resetAtEpoch,
|
|
321
|
+
resets_at: resetAt,
|
|
322
|
+
};
|
|
323
|
+
|
|
324
|
+
mirrorStore.writeProvider("glm", updated);
|
|
325
|
+
console.log(
|
|
326
|
+
`[GLMQuotaCountdown] Updated mirror with reset time: ${resetAt}`,
|
|
327
|
+
);
|
|
328
|
+
} catch (error) {
|
|
329
|
+
console.error(`[GLMQuotaCountdown] Failed to update mirror: ${error}`);
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* Schedule the countdown timer
|
|
335
|
+
*/
|
|
336
|
+
private scheduleCountdown(
|
|
337
|
+
jobId: string,
|
|
338
|
+
state: GLMQuotaCountdownState,
|
|
339
|
+
mirrorStore: MirrorStore,
|
|
340
|
+
machine: JobStateMachine,
|
|
341
|
+
): void {
|
|
342
|
+
const tickIntervalMs = FOOTER_UPDATE_INTERVAL_SECONDS * 1000;
|
|
343
|
+
|
|
344
|
+
// Schedule periodic ticks
|
|
345
|
+
const tickTimer = setInterval(() => {
|
|
346
|
+
this.handleTick(jobId, state, mirrorStore, machine);
|
|
347
|
+
}, tickIntervalMs);
|
|
348
|
+
|
|
349
|
+
this.timers.set(jobId, tickTimer);
|
|
350
|
+
|
|
351
|
+
// Calculate time until reset
|
|
352
|
+
const now = Date.now();
|
|
353
|
+
const timeUntilReset = state.resetAtEpoch - now;
|
|
354
|
+
|
|
355
|
+
// Schedule auto-resume just before reset
|
|
356
|
+
if (timeUntilReset > RESUME_BUFFER_MS + MIN_RESUME_DELAY_MS) {
|
|
357
|
+
const resumeDelay = Math.max(
|
|
358
|
+
timeUntilReset - RESUME_BUFFER_MS,
|
|
359
|
+
MIN_RESUME_DELAY_MS,
|
|
360
|
+
);
|
|
361
|
+
|
|
362
|
+
const resumeTimer = setTimeout(async () => {
|
|
363
|
+
await this.triggerAutoResume(jobId, state, machine, mirrorStore);
|
|
364
|
+
}, resumeDelay);
|
|
365
|
+
|
|
366
|
+
this.timers.set(`${jobId}:resume`, resumeTimer);
|
|
367
|
+
state.autoResumeScheduled = true;
|
|
368
|
+
} else if (timeUntilReset > 0) {
|
|
369
|
+
// Reset time is soon, schedule immediate resume
|
|
370
|
+
const resumeTimer = setTimeout(async () => {
|
|
371
|
+
await this.triggerAutoResume(jobId, state, machine, mirrorStore);
|
|
372
|
+
}, timeUntilReset);
|
|
373
|
+
|
|
374
|
+
this.timers.set(`${jobId}:resume`, resumeTimer);
|
|
375
|
+
state.autoResumeScheduled = true;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Handle a countdown tick
|
|
381
|
+
*/
|
|
382
|
+
private handleTick(
|
|
383
|
+
jobId: string,
|
|
384
|
+
state: GLMQuotaCountdownState,
|
|
385
|
+
mirrorStore: MirrorStore,
|
|
386
|
+
machine: JobStateMachine,
|
|
387
|
+
): void {
|
|
388
|
+
const now = Date.now();
|
|
389
|
+
const remainingMs = state.resetAtEpoch - now;
|
|
390
|
+
const remainingSeconds = Math.max(0, Math.floor(remainingMs / 1000));
|
|
391
|
+
|
|
392
|
+
// Check if it's reset time
|
|
393
|
+
if (remainingMs <= 0) {
|
|
394
|
+
this.triggerAutoResume(jobId, state, machine, mirrorStore);
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
// Check for notification intervals
|
|
399
|
+
this.checkNotificationIntervals(state, remainingSeconds);
|
|
400
|
+
|
|
401
|
+
// Emit tick for TUI updates
|
|
402
|
+
const nextNotificationIn = this.getNextNotificationIn(
|
|
403
|
+
state,
|
|
404
|
+
remainingSeconds,
|
|
405
|
+
);
|
|
406
|
+
this.emitTick({
|
|
407
|
+
jobId,
|
|
408
|
+
remainingSeconds,
|
|
409
|
+
remainingFormatted: formatCountdown(remainingSeconds),
|
|
410
|
+
nextNotificationIn,
|
|
411
|
+
isResetTime: false,
|
|
412
|
+
});
|
|
413
|
+
|
|
414
|
+
// Update countdown state
|
|
415
|
+
state.totalSeconds = remainingSeconds;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* Check if we need to send a notification
|
|
420
|
+
*/
|
|
421
|
+
private checkNotificationIntervals(
|
|
422
|
+
state: GLMQuotaCountdownState,
|
|
423
|
+
remainingSeconds: number,
|
|
424
|
+
): void {
|
|
425
|
+
for (const interval of NOTIFICATION_INTERVALS) {
|
|
426
|
+
if (
|
|
427
|
+
remainingSeconds <= interval &&
|
|
428
|
+
(!state.lastNotificationSent ||
|
|
429
|
+
this.isNewerNotification(state.lastNotificationSent, interval))
|
|
430
|
+
) {
|
|
431
|
+
this.sendCountdownNotification(state.jobId, remainingSeconds, interval);
|
|
432
|
+
state.lastNotificationSent = this.getNotificationType(interval);
|
|
433
|
+
break;
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/**
|
|
439
|
+
* Determine notification type from interval
|
|
440
|
+
*/
|
|
441
|
+
private getNotificationType(
|
|
442
|
+
interval: (typeof NOTIFICATION_INTERVALS)[number],
|
|
443
|
+
): "15min" | "5min" | "1min" | "reset" {
|
|
444
|
+
switch (interval) {
|
|
445
|
+
case 15 * 60:
|
|
446
|
+
return "15min";
|
|
447
|
+
case 5 * 60:
|
|
448
|
+
return "5min";
|
|
449
|
+
case 60:
|
|
450
|
+
return "1min";
|
|
451
|
+
default:
|
|
452
|
+
return "reset";
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/**
|
|
457
|
+
* Check if a notification type is "newer" (more urgent)
|
|
458
|
+
*/
|
|
459
|
+
private isNewerNotification(
|
|
460
|
+
current: "15min" | "5min" | "1min" | "reset",
|
|
461
|
+
newInterval: number,
|
|
462
|
+
): boolean {
|
|
463
|
+
const priority: Record<string, number> = {
|
|
464
|
+
"15min": 3,
|
|
465
|
+
"5min": 2,
|
|
466
|
+
"1min": 1,
|
|
467
|
+
reset: 0,
|
|
468
|
+
};
|
|
469
|
+
const newType = this.getNotificationType(
|
|
470
|
+
newInterval as (typeof NOTIFICATION_INTERVALS)[number],
|
|
471
|
+
);
|
|
472
|
+
return priority[newType] < priority[current];
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Get seconds until next notification
|
|
477
|
+
*/
|
|
478
|
+
private getNextNotificationIn(
|
|
479
|
+
state: GLMQuotaCountdownState,
|
|
480
|
+
remainingSeconds: number,
|
|
481
|
+
): number | undefined {
|
|
482
|
+
for (const interval of NOTIFICATION_INTERVALS) {
|
|
483
|
+
if (remainingSeconds <= interval && remainingSeconds > interval - 60) {
|
|
484
|
+
return remainingSeconds;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
return undefined;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
/**
|
|
491
|
+
* Send a countdown notification
|
|
492
|
+
*/
|
|
493
|
+
private async sendCountdownNotification(
|
|
494
|
+
jobId: string,
|
|
495
|
+
remainingSeconds: number,
|
|
496
|
+
interval: number,
|
|
497
|
+
): Promise<void> {
|
|
498
|
+
if (!this.notificationCenter) {
|
|
499
|
+
console.log(
|
|
500
|
+
`[GLMQuotaCountdown] Notification: ${formatCountdown(remainingSeconds)} until GLM quota reset`,
|
|
501
|
+
);
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
const minutes = Math.floor(remainingSeconds / 60);
|
|
506
|
+
const label =
|
|
507
|
+
interval === 15 * 60
|
|
508
|
+
? "15 minutes"
|
|
509
|
+
: interval === 5 * 60
|
|
510
|
+
? "5 minutes"
|
|
511
|
+
: "1 minute";
|
|
512
|
+
|
|
513
|
+
try {
|
|
514
|
+
await this.notificationCenter.notify("QuotaPaused", {
|
|
515
|
+
jobId: this.jobContext?.jobId ?? jobId,
|
|
516
|
+
requirement: this.jobContext?.requirement ?? "GLM Quota",
|
|
517
|
+
error: `GLM quota reset in ${label} (${formatCountdown(remainingSeconds)}). Auto-resume pending.`,
|
|
518
|
+
});
|
|
519
|
+
console.log(
|
|
520
|
+
`[GLMQuotaCountdown] Sent ${label} notification for job ${jobId}`,
|
|
521
|
+
);
|
|
522
|
+
} catch (error) {
|
|
523
|
+
console.error(`[GLMQuotaCountdown] Failed to send notification: ${error}`);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/**
|
|
528
|
+
* Trigger auto-resume when quota resets
|
|
529
|
+
*/
|
|
530
|
+
private async triggerAutoResume(
|
|
531
|
+
jobId: string,
|
|
532
|
+
state: GLMQuotaCountdownState,
|
|
533
|
+
machine: JobStateMachine,
|
|
534
|
+
mirrorStore: MirrorStore,
|
|
535
|
+
): Promise<void> {
|
|
536
|
+
console.log(`[GLMQuotaCountdown] Triggering auto-resume for job ${jobId}`);
|
|
537
|
+
|
|
538
|
+
// Clear timers
|
|
539
|
+
this.cancelCountdown(jobId);
|
|
540
|
+
|
|
541
|
+
// Send final notification
|
|
542
|
+
await this.sendCountdownNotification(jobId, 0, 0);
|
|
543
|
+
|
|
544
|
+
// Attempt to transition machine back to running
|
|
545
|
+
try {
|
|
546
|
+
const checkpoint = machine.getCheckpoint();
|
|
547
|
+
if (checkpoint && checkpoint.status === "paused_quota") {
|
|
548
|
+
const result = await machine.transition("running");
|
|
549
|
+
if (result.success) {
|
|
550
|
+
console.log(`[GLMQuotaCountdown] Job ${jobId} auto-resumed successfully`);
|
|
551
|
+
|
|
552
|
+
// Update mirror to clear exhaustion
|
|
553
|
+
const record = mirrorStore.readProvider("glm");
|
|
554
|
+
if (record) {
|
|
555
|
+
mirrorStore.writeProvider("glm", {
|
|
556
|
+
...record,
|
|
557
|
+
synced_at: new Date().toISOString(),
|
|
558
|
+
exhausted: false,
|
|
559
|
+
h5_used_pct: 0,
|
|
560
|
+
resets_at: undefined,
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// Emit final tick
|
|
565
|
+
this.emitTick({
|
|
566
|
+
jobId,
|
|
567
|
+
remainingSeconds: 0,
|
|
568
|
+
remainingFormatted: "RESET - RESUMING",
|
|
569
|
+
isResetTime: true,
|
|
570
|
+
});
|
|
571
|
+
} else {
|
|
572
|
+
console.error(`[GLMQuotaCountdown] Auto-resume failed: ${result.error}`);
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
} catch (error) {
|
|
576
|
+
console.error(`[GLMQuotaCountdown] Auto-resume error: ${error}`);
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
/**
|
|
581
|
+
* Emit tick to all callbacks
|
|
582
|
+
*/
|
|
583
|
+
private emitTick(event: CountdownTickEvent): void {
|
|
584
|
+
for (const callback of this.tickCallbacks) {
|
|
585
|
+
try {
|
|
586
|
+
callback(event);
|
|
587
|
+
} catch (error) {
|
|
588
|
+
console.error(`[GLMQuotaCountdown] Tick callback error: ${error}`);
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
/**
|
|
594
|
+
* Clean up all timers
|
|
595
|
+
*/
|
|
596
|
+
dispose(): void {
|
|
597
|
+
for (const timer of this.timers.values()) {
|
|
598
|
+
clearTimeout(timer);
|
|
599
|
+
clearInterval(timer);
|
|
600
|
+
}
|
|
601
|
+
this.timers.clear();
|
|
602
|
+
this.activeCountdowns.clear();
|
|
603
|
+
this.tickCallbacks.clear();
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/** Singleton instance for global access */
|
|
608
|
+
let globalCountdown: GLMQuotaCountdown | null = null;
|
|
609
|
+
|
|
610
|
+
export function getGLMQuotaCountdown(): GLMQuotaCountdown {
|
|
611
|
+
if (!globalCountdown) {
|
|
612
|
+
globalCountdown = new GLMQuotaCountdown();
|
|
613
|
+
}
|
|
614
|
+
return globalCountdown;
|
|
615
|
+
}
|
package/harness/index.ts
CHANGED
|
@@ -210,6 +210,18 @@ export type {
|
|
|
210
210
|
LearnedExperience,
|
|
211
211
|
} from "../packages/learning-engine/src/types.js";
|
|
212
212
|
|
|
213
|
+
// --- GLM Quota Countdown -----------------------------------------------
|
|
214
|
+
export {
|
|
215
|
+
GLMQuotaCountdown,
|
|
216
|
+
getGLMQuotaCountdown,
|
|
217
|
+
parseGLMResetTime,
|
|
218
|
+
formatCountdown,
|
|
219
|
+
} from "./glm-quota-countdown.js";
|
|
220
|
+
export type {
|
|
221
|
+
GLMQuotaCountdownState,
|
|
222
|
+
CountdownTickEvent,
|
|
223
|
+
} from "./glm-quota-countdown.js";
|
|
224
|
+
|
|
213
225
|
// --- RFC-0059: Experience Replay ------------------------------------------
|
|
214
226
|
export {
|
|
215
227
|
ExperienceReplay,
|
package/package.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tui-usage-monitor.d.ts","sourceRoot":"","sources":["../src/tui-usage-monitor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,QAAQ,GAAG,KAAK,GAAG,WAAW,GAAG,YAAY,GAAG,SAAS,CAAC;IACpE,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mCAAmC;IACnC,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,iBAAiB;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0BAA0B;IAC1B,SAAS,EAAE,QAAQ,GAAG,gBAAgB,GAAG,YAAY,GAAG,SAAS,CAAC;IAClE,uBAAuB;IACvB,eAAe,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,qBAAqB;IACrC,iDAAiD;IACjD,YAAY,EAAE,YAAY,CAAC;IAC3B,2BAA2B;IAC3B,KAAK,CAAC,EAAE,OAAO,CAAC;CAChB;
|
|
1
|
+
{"version":3,"file":"tui-usage-monitor.d.ts","sourceRoot":"","sources":["../src/tui-usage-monitor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,QAAQ,GAAG,KAAK,GAAG,WAAW,GAAG,YAAY,GAAG,SAAS,CAAC;IACpE,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mCAAmC;IACnC,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,iBAAiB;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0BAA0B;IAC1B,SAAS,EAAE,QAAQ,GAAG,gBAAgB,GAAG,YAAY,GAAG,SAAS,CAAC;IAClE,uBAAuB;IACvB,eAAe,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,qBAAqB;IACrC,iDAAiD;IACjD,YAAY,EAAE,YAAY,CAAC;IAC3B,2BAA2B;IAC3B,KAAK,CAAC,EAAE,OAAO,CAAC;CAChB;AAsDD,qBAAa,eAAgB,SAAQ,YAAY;IAChD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAU;IAChC,OAAO,CAAC,WAAW,CAA0C;IAE7D,YAAY,MAAM,EAAE,qBAAqB,EAIxC;IAED;;OAEG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI,CA4BrD;IAED;;OAEG;IACH,cAAc,CACb,OAAO,EAAE,MAAM,GACb,QAAQ,GAAG,KAAK,GAAG,WAAW,GAAG,YAAY,GAAG,IAAI,CAStD;IAED;;OAEG;IACH,aAAa,CACZ,QAAQ,EAAE,QAAQ,GAAG,KAAK,GAAG,WAAW,GAAG,YAAY,GACrD,cAAc,GAAG,IAAI,CAEvB;IAED;;OAEG;IACH,iBAAiB,IAAI,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAE/C;IAED;;OAEG;IACH,KAAK,IAAI,IAAI,CAEZ;IAED;;OAEG;IACH,OAAO,CAAC,WAAW;IAqCnB;;OAEG;IACH,OAAO,CAAC,eAAe;IAsCvB;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAcjC;;OAEG;IACH,OAAO,CAAC,cAAc;IA2CtB;;OAEG;IACH,OAAO,CAAC,oBAAoB;CAmB5B;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACpC,YAAY,EAAE,YAAY,EAC1B,EAAE,EAAE;IACH,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,KAAK,IAAI,CAAC;CAC3E,EACD,MAAM,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,GAC1B,eAAe,CAgBjB"}
|
|
@@ -36,6 +36,10 @@ const PROVIDER_PATTERNS = {
|
|
|
36
36
|
/zhipu.*(?:quota|limit|exhausted)/i,
|
|
37
37
|
/glm.*context.*(?:window|length)/i,
|
|
38
38
|
/glm.*rate.?limit/i,
|
|
39
|
+
/glm.*429/i,
|
|
40
|
+
/code.*1308/i, // GLM specific error code
|
|
41
|
+
/Usage limit reached/i,
|
|
42
|
+
/5 hour.*quota/i,
|
|
39
43
|
],
|
|
40
44
|
anthropic: [
|
|
41
45
|
/(?:^|\s)Claude(?:$|\s)/i,
|
|
@@ -51,6 +55,8 @@ const PROVIDER_PATTERNS = {
|
|
|
51
55
|
],
|
|
52
56
|
};
|
|
53
57
|
const RESET_TIME_PATTERNS = [
|
|
58
|
+
// GLM specific: "reset at 2026-08-25 01:47:16" or "reset at 2026-08-25T01:47:16"
|
|
59
|
+
/reset\s+(?:at\s+)?(\d{4}-\d{2}-\d{2}[T ]?\d{2}:\d{2}:\d{2})/i,
|
|
54
60
|
/reset(?:s)?\s+(?:at|in)\s+(\d{1,2}):(\d{2})/i,
|
|
55
61
|
/(?:at|in)\s+(\d{1,2}):(\d{2})/i,
|
|
56
62
|
/retry\s+after\s+(\d{1,2}):(\d{2})/i,
|
|
@@ -134,9 +140,9 @@ export class TUIUsageMonitor extends EventEmitter {
|
|
|
134
140
|
const pctMatch = message.match(PERCENTAGE_PATTERN);
|
|
135
141
|
const usedPct = pctMatch ? parseInt(pctMatch[1], 10) : undefined;
|
|
136
142
|
// Determine if exhausted
|
|
137
|
-
const exhausted = usedPct
|
|
138
|
-
?
|
|
139
|
-
:
|
|
143
|
+
const exhausted = usedPct === undefined
|
|
144
|
+
? this.containsExhaustedKeywords(message)
|
|
145
|
+
: usedPct >= 100;
|
|
140
146
|
if (!exhausted && !resetsAt) {
|
|
141
147
|
// Not a quota signal
|
|
142
148
|
return null;
|
|
@@ -145,7 +151,7 @@ export class TUIUsageMonitor extends EventEmitter {
|
|
|
145
151
|
provider,
|
|
146
152
|
timestamp: new Date().toISOString(),
|
|
147
153
|
usedPct,
|
|
148
|
-
remainingPct: usedPct
|
|
154
|
+
remainingPct: usedPct === undefined ? 0 : 100 - usedPct,
|
|
149
155
|
exhausted,
|
|
150
156
|
resetsAt,
|
|
151
157
|
limitType,
|
|
@@ -203,6 +209,16 @@ export class TUIUsageMonitor extends EventEmitter {
|
|
|
203
209
|
for (const pattern of RESET_TIME_PATTERNS) {
|
|
204
210
|
const match = message.match(pattern);
|
|
205
211
|
if (match) {
|
|
212
|
+
// Check if this is the full datetime pattern (first in list)
|
|
213
|
+
if (pattern === RESET_TIME_PATTERNS[0] && match[1].includes("-")) {
|
|
214
|
+
// GLM full datetime format: "2026-08-25 01:47:16" or "2026-08-25T01:47:16"
|
|
215
|
+
const datetimeStr = match[1].replace(" ", "T");
|
|
216
|
+
const date = new Date(datetimeStr);
|
|
217
|
+
if (!isNaN(date.getTime())) {
|
|
218
|
+
return date.toISOString();
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
// Time-only patterns
|
|
206
222
|
let hour = parseInt(match[1], 10);
|
|
207
223
|
const minute = parseInt(match[2], 10);
|
|
208
224
|
// Handle AM/PM
|