twinclaw 1.0.0

package/dist/services/runtime-budget-governor.js

@@ -0,0 +1,415 @@
+import { randomUUID } from 'node:crypto';
+import { clearRuntimeBudgetState, getRuntimeBudgetState, getRuntimeDailyUsageAggregate, getRuntimeSessionUsageAggregate, listRuntimeBudgetEvents, listRuntimeProviderUsageAggregates, recordRuntimeBudgetEvent, recordRuntimeUsageEvent, setRuntimeBudgetState, } from './db.js';
+import { logThought } from '../utils/logger.js';
+import { getConfigValue } from '../config/config-loader.js';
+const MANUAL_PROFILE_KEY = 'manual_profile';
+const DEFAULT_SESSION_ID = 'global';
+const DEFAULT_LOCAL_MODEL_ID = 'local';
+const DEFAULT_LIMITS = {
+    dailyRequestLimit: 2_400,
+    dailyTokenLimit: 5_000_000,
+    sessionRequestLimit: 300,
+    sessionTokenLimit: 700_000,
+    providerRequestLimit: 1_000,
+    providerTokenLimit: 2_200_000,
+    warningRatio: 0.8,
+    warningPacingMs: 250,
+    hardLimitPacingMs: 1_250,
+    providerCooldownMs: 60_000,
+};
+const PROFILE_ORDER = ['economy', 'balanced', 'performance'];
+export class RuntimeBudgetGovernor {
+    #limits;
+    #defaultProfile;
+    #preferLocalModel;
+    #localModelId;
+    #now;
+    #providerCooldowns = new Map();
+    #signatures = new Map();
+    #manualProfile;
+    constructor(config = {}) {
+        this.#limits = resolveLimits(config.limits);
+        this.#defaultProfile =
+            parseProfile(getConfigValue('RUNTIME_BUDGET_DEFAULT_PROFILE')) ??
+                config.defaultProfile ??
+                'performance';
+        this.#preferLocalModel =
+            config.preferLocalModel ??
+                parseBoolean(getConfigValue('RUNTIME_BUDGET_PREFER_LOCAL_MODEL')) ??
+                false;
+        this.#localModelId =
+            getConfigValue('RUNTIME_BUDGET_LOCAL_MODEL_ID')?.trim() || config.localModelId || DEFAULT_LOCAL_MODEL_ID;
+        this.#now = config.now ?? (() => Date.now());
+        this.#manualProfile = parseProfile(getRuntimeBudgetState(MANUAL_PROFILE_KEY));
+    }
+    get limits() {
+        return this.#limits;
+    }
+    getRoutingDirective(sessionId) {
+        const normalizedSessionId = normalizeSessionId(sessionId);
+        const evaluation = this.#evaluate(normalizedSessionId);
+        this.#recordTransition(normalizedSessionId, evaluation);
+        return evaluation.directive;
+    }
+    recordUsage(input) {
+        const normalizedSessionId = normalizeSessionId(input.sessionId);
+        const event = {
+            id: randomUUID(),
+            sessionId: normalizedSessionId,
+            modelId: input.modelId,
+            providerId: input.providerId,
+            profile: input.profile,
+            stage: input.stage,
+            requestTokens: Math.max(0, Math.floor(input.requestTokens)),
+            responseTokens: Math.max(0, Math.floor(input.responseTokens)),
+            latencyMs: Math.max(0, Math.floor(input.latencyMs)),
+            statusCode: input.statusCode ?? null,
+            error: input.error ?? null,
+        };
+        recordRuntimeUsageEvent(event);
+        if (input.stage === 'failure' && input.statusCode === 429) {
+            this.applyProviderCooldown(input.providerId, normalizedSessionId, 'Rate-limit response detected.');
+        }
+    }
+    applyProviderCooldown(providerId, sessionId, reason = 'Provider cooldown applied.') {
+        const normalizedProvider = providerId.trim().toLowerCase();
+        if (!normalizedProvider) {
+            return;
+        }
+        const now = this.#now();
+        const current = this.#providerCooldowns.get(normalizedProvider) ?? 0;
+        const next = Math.max(current, now + this.#limits.providerCooldownMs);
+        this.#providerCooldowns.set(normalizedProvider, next);
+        const normalizedSessionId = normalizeSessionId(sessionId);
+        this.#recordBudgetEvent({
+            sessionId: normalizedSessionId,
+            severity: 'warning',
+            profile: this.#manualProfile ?? this.#defaultProfile,
+            action: 'provider_cooldown',
+            reason,
+            detail: {
+                providerId: normalizedProvider,
+                cooldownUntil: new Date(next).toISOString(),
+            },
+        });
+    }
+    setManualProfile(profile, sessionId) {
+        const normalizedSessionId = normalizeSessionId(sessionId);
+        this.#manualProfile = profile;
+        if (profile) {
+            setRuntimeBudgetState(MANUAL_PROFILE_KEY, profile);
+        }
+        else {
+            clearRuntimeBudgetState(MANUAL_PROFILE_KEY);
+        }
+        this.#recordBudgetEvent({
+            sessionId: normalizedSessionId,
+            severity: 'ok',
+            profile: profile ?? this.#defaultProfile,
+            action: 'none',
+            reason: profile
+                ? `Manual budget profile override set to '${profile}'.`
+                : 'Manual budget profile override cleared.',
+            detail: { manualProfile: profile },
+        });
+    }
+    resetPolicyState(sessionId) {
+        const normalizedSessionId = normalizeSessionId(sessionId);
+        this.#providerCooldowns.clear();
+        this.#signatures.clear();
+        this.setManualProfile(null, normalizedSessionId);
+        this.#recordBudgetEvent({
+            sessionId: normalizedSessionId,
+            severity: 'ok',
+            profile: this.#defaultProfile,
+            action: 'none',
+            reason: 'Runtime budget policy state reset.',
+            detail: {},
+        });
+    }
+    getSnapshot(sessionId, eventLimit = 50) {
+        const normalizedSessionId = normalizeSessionId(sessionId);
+        const evaluation = this.#evaluate(normalizedSessionId);
+        return {
+            sessionId: normalizedSessionId,
+            manualProfile: evaluation.manualProfile,
+            limits: evaluation.limits,
+            daily: evaluation.daily,
+            session: evaluation.session,
+            providers: evaluation.providers,
+            directive: evaluation.directive,
+            recentEvents: this.getRecentEvents(eventLimit),
+        };
+    }
+    getRecentEvents(limit = 100) {
+        return listRuntimeBudgetEvents(limit).map((row) => this.#toEvent(row));
+    }
+    #evaluate(sessionId) {
+        this.#pruneCooldowns();
+        this.#manualProfile = parseProfile(getRuntimeBudgetState(MANUAL_PROFILE_KEY));
+        const daily = toUsageAggregate(getRuntimeDailyUsageAggregate());
+        const session = toUsageAggregate(getRuntimeSessionUsageAggregate(sessionId));
+        const providers = listRuntimeProviderUsageAggregates().map((row) => ({
+            providerId: row.provider_id,
+            ...toUsageAggregate(row),
+        }));
+        const severity = resolveSeverity(this.#limits, daily, session, providers);
+        const actions = resolveActions(severity, this.#preferLocalModel);
+        const blockedProviders = providers
+            .filter((provider) => provider.requestCount >= this.#limits.providerRequestLimit ||
+            provider.requestTokens >= this.#limits.providerTokenLimit)
+            .map((provider) => provider.providerId);
+        for (const providerId of blockedProviders) {
+            if (!this.#providerCooldowns.has(providerId)) {
+                this.#providerCooldowns.set(providerId, this.#now() + this.#limits.providerCooldownMs);
+            }
+        }
+        const cooldownProviders = [...this.#providerCooldowns.entries()]
+            .filter(([, until]) => until > this.#now())
+            .map(([providerId]) => providerId);
+        const finalBlockedProviders = [...new Set([...blockedProviders, ...cooldownProviders])];
+        if (finalBlockedProviders.length > 0 && !actions.includes('provider_cooldown')) {
+            actions.push('provider_cooldown');
+        }
+        const profile = this.#manualProfile ?? resolveProfile(this.#defaultProfile, severity);
+        const blockedModelIds = severity === 'hard_limit'
+            ? ['primary']
+            : this.#preferLocalModel && profile === 'economy'
+                ? ['primary', 'fallback_1', 'fallback_2'].filter((id) => id !== this.#localModelId)
+                : [];
+        const pacingDelayMs = severity === 'hard_limit'
+            ? this.#limits.hardLimitPacingMs
+            : severity === 'warning'
+                ? this.#limits.warningPacingMs
+                : 0;
+        const directive = {
+            profile,
+            severity,
+            actions,
+            pacingDelayMs,
+            blockedModelIds,
+            blockedProviders: finalBlockedProviders,
+            reason: buildReason(this.#limits, severity, daily, session, providers),
+            evaluatedAt: new Date(this.#now()).toISOString(),
+        };
+        return {
+            manualProfile: this.#manualProfile,
+            limits: this.#limits,
+            daily,
+            session,
+            providers,
+            directive,
+        };
+    }
+    #recordTransition(sessionId, evaluation) {
+        const signature = [
+            evaluation.directive.severity,
+            evaluation.directive.profile,
+            evaluation.directive.actions.join(','),
+            evaluation.directive.blockedProviders.join(','),
+            evaluation.directive.blockedModelIds.join(','),
+        ].join('|');
+        const previous = this.#signatures.get(sessionId);
+        if (previous === signature) {
+            return;
+        }
+        this.#signatures.set(sessionId, signature);
+        this.#recordBudgetEvent({
+            sessionId,
+            severity: evaluation.directive.severity,
+            profile: evaluation.directive.profile,
+            action: evaluation.directive.actions[0] ?? 'none',
+            reason: evaluation.directive.reason,
+            detail: {
+                actions: evaluation.directive.actions,
+                blockedProviders: evaluation.directive.blockedProviders,
+                blockedModelIds: evaluation.directive.blockedModelIds,
+                pacingDelayMs: evaluation.directive.pacingDelayMs,
+            },
+        });
+    }
+    #recordBudgetEvent(input) {
+        recordRuntimeBudgetEvent({
+            id: randomUUID(),
+            sessionId: input.sessionId,
+            severity: input.severity,
+            profile: input.profile,
+            action: input.action,
+            reason: input.reason,
+            detailJson: JSON.stringify(input.detail),
+        });
+        void logThought(`[RuntimeBudget] severity=${input.severity} profile=${input.profile} action=${input.action} reason=${input.reason}`);
+    }
+    #toEvent(row) {
+        return {
+            id: row.id,
+            sessionId: row.session_id,
+            severity: row.severity,
+            profile: row.profile,
+            action: row.action,
+            reason: row.reason,
+            detail: parseDetail(row.detail_json),
+            createdAt: row.created_at,
+        };
+    }
+    #pruneCooldowns() {
+        const now = this.#now();
+        for (const [providerId, cooldownUntil] of this.#providerCooldowns.entries()) {
+            if (cooldownUntil <= now) {
+                this.#providerCooldowns.delete(providerId);
+            }
+        }
+    }
+}
+function normalizeSessionId(value) {
+    const normalized = value?.trim();
+    return normalized || DEFAULT_SESSION_ID;
+}
+function toUsageAggregate(row) {
+    return {
+        requestCount: Number(row.request_count ?? 0),
+        requestTokens: Number(row.request_tokens ?? 0),
+        responseTokens: Number(row.response_tokens ?? 0),
+        failureCount: Number(row.failure_count ?? 0),
+        skippedCount: Number(row.skipped_count ?? 0),
+    };
+}
+function resolveProfile(defaultProfile, severity) {
+    if (severity === 'hard_limit') {
+        return 'economy';
+    }
+    if (severity === 'warning' && defaultProfile === 'performance') {
+        return 'balanced';
+    }
+    return defaultProfile;
+}
+function resolveSeverity(limits, daily, session, providers) {
+    const hardBreached = daily.requestCount >= limits.dailyRequestLimit ||
+        daily.requestTokens >= limits.dailyTokenLimit ||
+        session.requestCount >= limits.sessionRequestLimit ||
+        session.requestTokens >= limits.sessionTokenLimit ||
+        providers.some((provider) => provider.requestCount >= limits.providerRequestLimit ||
+            provider.requestTokens >= limits.providerTokenLimit);
+    if (hardBreached) {
+        return 'hard_limit';
+    }
+    const ratios = [
+        safeRatio(daily.requestCount, limits.dailyRequestLimit),
+        safeRatio(daily.requestTokens, limits.dailyTokenLimit),
+        safeRatio(session.requestCount, limits.sessionRequestLimit),
+        safeRatio(session.requestTokens, limits.sessionTokenLimit),
+        ...providers.flatMap((provider) => [
+            safeRatio(provider.requestCount, limits.providerRequestLimit),
+            safeRatio(provider.requestTokens, limits.providerTokenLimit),
+        ]),
+    ];
+    if (ratios.some((ratio) => ratio >= limits.warningRatio)) {
+        return 'warning';
+    }
+    return 'ok';
+}
+function resolveActions(severity, preferLocalModel) {
+    if (severity === 'ok') {
+        return ['none'];
+    }
+    const actions = ['intelligent_pacing'];
+    if (severity === 'hard_limit') {
+        actions.push('fallback_tightening');
+        if (preferLocalModel) {
+            actions.push('prefer_local_model');
+        }
+    }
+    return actions;
+}
+function buildReason(limits, severity, daily, session, providers) {
+    if (severity === 'ok') {
+        return 'Budget utilization is below warning thresholds.';
+    }
+    const providerHeavy = providers
+        .filter((provider) => provider.requestCount >= limits.providerRequestLimit * limits.warningRatio ||
+        provider.requestTokens >= limits.providerTokenLimit * limits.warningRatio)
+        .map((provider) => provider.providerId);
+    const parts = [
+        `dailyRequests=${daily.requestCount}/${limits.dailyRequestLimit}`,
+        `dailyTokens=${daily.requestTokens}/${limits.dailyTokenLimit}`,
+        `sessionRequests=${session.requestCount}/${limits.sessionRequestLimit}`,
+        `sessionTokens=${session.requestTokens}/${limits.sessionTokenLimit}`,
+    ];
+    if (providerHeavy.length > 0) {
+        parts.push(`providers=${providerHeavy.join(',')}`);
+    }
+    return severity === 'hard_limit'
+        ? `Hard budget threshold reached (${parts.join(' | ')}).`
+        : `Warning budget threshold reached (${parts.join(' | ')}).`;
+}
+function safeRatio(value, limit) {
+    if (!Number.isFinite(limit) || limit <= 0) {
+        return 0;
+    }
+    return value / limit;
+}
+function parseDetail(value) {
+    try {
+        const parsed = JSON.parse(value);
+        return parsed ?? {};
+    }
+    catch {
+        return {};
+    }
+}
+function resolveLimits(overrides = {}) {
+    return {
+        dailyRequestLimit: readIntEnv('RUNTIME_BUDGET_DAILY_REQUEST_LIMIT', overrides.dailyRequestLimit ?? DEFAULT_LIMITS.dailyRequestLimit),
+        dailyTokenLimit: readIntEnv('RUNTIME_BUDGET_DAILY_TOKEN_LIMIT', overrides.dailyTokenLimit ?? DEFAULT_LIMITS.dailyTokenLimit),
+        sessionRequestLimit: readIntEnv('RUNTIME_BUDGET_SESSION_REQUEST_LIMIT', overrides.sessionRequestLimit ?? DEFAULT_LIMITS.sessionRequestLimit),
+        sessionTokenLimit: readIntEnv('RUNTIME_BUDGET_SESSION_TOKEN_LIMIT', overrides.sessionTokenLimit ?? DEFAULT_LIMITS.sessionTokenLimit),
+        providerRequestLimit: readIntEnv('RUNTIME_BUDGET_PROVIDER_REQUEST_LIMIT', overrides.providerRequestLimit ?? DEFAULT_LIMITS.providerRequestLimit),
+        providerTokenLimit: readIntEnv('RUNTIME_BUDGET_PROVIDER_TOKEN_LIMIT', overrides.providerTokenLimit ?? DEFAULT_LIMITS.providerTokenLimit),
+        warningRatio: readFloatEnv('RUNTIME_BUDGET_WARNING_RATIO', overrides.warningRatio ?? DEFAULT_LIMITS.warningRatio),
+        warningPacingMs: readIntEnv('RUNTIME_BUDGET_WARNING_PACING_MS', overrides.warningPacingMs ?? DEFAULT_LIMITS.warningPacingMs),
+        hardLimitPacingMs: readIntEnv('RUNTIME_BUDGET_HARD_PACING_MS', overrides.hardLimitPacingMs ?? DEFAULT_LIMITS.hardLimitPacingMs),
+        providerCooldownMs: readIntEnv('RUNTIME_BUDGET_PROVIDER_COOLDOWN_MS', overrides.providerCooldownMs ?? DEFAULT_LIMITS.providerCooldownMs),
+    };
+}
+function readIntEnv(name, fallback) {
+    const raw = getConfigValue(name);
+    if (!raw) {
+        return fallback;
+    }
+    const parsed = Number(raw);
+    if (!Number.isFinite(parsed)) {
+        return fallback;
+    }
+    return Math.max(1, Math.floor(parsed));
+}
+function readFloatEnv(name, fallback) {
+    const raw = getConfigValue(name);
+    if (!raw) {
+        return fallback;
+    }
+    const parsed = Number(raw);
+    if (!Number.isFinite(parsed)) {
+        return fallback;
+    }
+    return Math.min(0.99, Math.max(0.01, parsed));
+}
+function parseBoolean(raw) {
+    if (!raw) {
+        return null;
+    }
+    const normalized = raw.trim().toLowerCase();
+    if (['1', 'true', 'yes', 'on'].includes(normalized)) {
+        return true;
+    }
+    if (['0', 'false', 'no', 'off'].includes(normalized)) {
+        return false;
+    }
+    return null;
+}
+function parseProfile(raw) {
+    if (!raw) {
+        return null;
+    }
+    const normalized = raw.trim().toLowerCase();
+    return PROFILE_ORDER.find((profile) => profile === normalized) ?? null;
+}