thumbgate 1.16.12 → 1.16.19

package/scripts/deepseek-v4-runtime-guardrails.js

@@ -0,0 +1,253 @@
+#!/usr/bin/env node
+'use strict';
+
+const { listGateTemplates } = require('./gate-templates');
+
+const CATEGORY = 'Sparse Attention Runtime Safety';
+
+function normalizeBoolean(value) {
+  if (value === true) return true;
+  if (value === false || value === undefined || value === null) return false;
+  return /^(1|true|yes|on)$/i.test(String(value).trim());
+}
+
+function toNumber(value) {
+  if (value === undefined || value === null || value === '') return null;
+  const num = Number(value);
+  return Number.isFinite(num) ? num : null;
+}
+
+function normalizeOptions(options = {}) {
+  const training = normalizeBoolean(options.training || options.rl || options['verified-rl']);
+  const kvOffload = normalizeBoolean(options['kv-offload'] || options['cpu-kv-offload'] || options.hisparse);
+  return {
+    workload: String(options.workload || options.name || 'deepseek-v4-runtime').trim() || 'deepseek-v4-runtime',
+    model: String(options.model || 'deepseek-v4-flash').trim() || 'deepseek-v4-flash',
+    engine: String(options.engine || 'sglang').trim() || 'sglang',
+    contextTokens: toNumber(options['context-tokens'] || options.context),
+    targetContextTokens: toNumber(options['target-context-tokens'] || options.target) || 1000000,
+    baselineThroughput: toNumber(options['baseline-throughput'] || options['baseline-tps']),
+    newThroughput: toNumber(options['new-throughput'] || options['new-tps']),
+    hybridAttention: normalizeBoolean(options['hybrid-attention'] || options.hybrid),
+    prefixCache: normalizeBoolean(options['prefix-cache'] || options.shadowradix),
+    cacheCoherenceEval: normalizeBoolean(options['cache-coherence-eval'] || options['cache-eval']),
+    speculativeDecoding: normalizeBoolean(options['speculative-decoding'] || options.speculative || options.mtp || options.eagle),
+    acceptLength: toNumber(options['accept-length'] || options['spec-accept-length']),
+    kvOffload,
+    training,
+    rolloutReplay: normalizeBoolean(options['rollout-replay'] || options.r3),
+    indexerReplay: normalizeBoolean(options['indexer-replay']),
+    trainInferenceDrift: toNumber(options['train-inference-drift'] || options.drift),
+    precisionMode: String(options['precision-mode'] || options.precision || '').trim().toLowerCase(),
+    deterministic: normalizeBoolean(options.deterministic || options['deterministic-kernels']),
+    numericalSpikes: normalizeBoolean(options['numerical-spikes'] || options['kl-spikes']),
+  };
+}
+
+function throughputDropPercent(options) {
+  if (options.baselineThroughput === null || options.newThroughput === null || options.baselineThroughput <= 0) return null;
+  return Number((((options.baselineThroughput - options.newThroughput) / options.baselineThroughput) * 100).toFixed(2));
+}
+
+function isLongContext(options) {
+  const context = options.contextTokens || options.targetContextTokens;
+  return context >= 128000;
+}
+
+function usesMixedPrecision(options) {
+  return /fp4|fp8|mxfp|mixed/.test(options.precisionMode);
+}
+
+function templateApplicability(template, options) {
+  const drop = throughputDropPercent(options);
+  if (template.id === 'require-hybrid-prefix-cache-coherence-eval') {
+    return (options.hybridAttention || isLongContext(options)) && (!options.prefixCache || !options.cacheCoherenceEval);
+  }
+  if (template.id === 'checkpoint-speculative-decoding-acceptance') {
+    return options.speculativeDecoding && (options.acceptLength === null || options.acceptLength < 2 || !options.cacheCoherenceEval);
+  }
+  if (template.id === 'require-long-context-kv-offload-capacity-plan') {
+    return isLongContext(options) && !options.kvOffload;
+  }
+  if (template.id === 'require-rollout-routing-and-indexer-replay') {
+    return options.training && (!options.rolloutReplay || !options.indexerReplay || (options.trainInferenceDrift !== null && options.trainInferenceDrift > 0.05));
+  }
+  if (template.id === 'checkpoint-mixed-precision-determinism') {
+    return (usesMixedPrecision(options) || options.numericalSpikes) && !options.deterministic;
+  }
+  if (template.id === 'checkpoint-long-context-throughput-regression') {
+    return drop !== null && drop > 10;
+  }
+  return false;
+}
+
+function buildSignals(options) {
+  const drop = throughputDropPercent(options);
+  return [
+    hybridAttentionSignal(options),
+    speculativeDecodingSignal(options),
+    longContextSignal(options, drop),
+    verifiedReplaySignal(options),
+    mixedPrecisionSignal(options),
+  ].filter(Boolean);
+}
+
+function hybridAttentionSignal(options) {
+  if (!(options.hybridAttention || isLongContext(options) || options.prefixCache)) return null;
+  return {
+    id: 'hybrid_attention_cache',
+    label: 'Hybrid attention prefix cache',
+    values: [
+      options.hybridAttention ? 'hybrid attention' : null,
+      options.prefixCache ? 'prefix cache enabled' : 'prefix cache missing',
+      options.cacheCoherenceEval ? 'coherence eval present' : 'missing coherence eval',
+      options.contextTokens !== null ? `${options.contextTokens} context tokens` : null,
+    ].filter(Boolean),
+    risk: 'SWA, compressed KV, and compression-state pools can drift unless cache lifetime and reuse are verified.',
+  };
+}
+
+function speculativeDecodingSignal(options) {
+  if (!(options.speculativeDecoding || options.acceptLength !== null)) return null;
+  return {
+    id: 'speculative_decoding',
+    label: 'Speculative decoding rollout',
+    values: [
+      options.speculativeDecoding ? 'speculative decoding enabled' : 'speculative decoding not declared',
+      options.acceptLength !== null ? `${options.acceptLength} accept length` : 'accept length missing',
+    ],
+    risk: 'Draft-token metadata and rollback paths can make throughput claims look good while correctness or acceptance collapses.',
+  };
+}
+
+function longContextSignal(options, drop) {
+  if (!isLongContext(options)) return null;
+  return {
+    id: 'long_context_capacity',
+    label: 'Long-context capacity plan',
+    values: [
+      `${options.contextTokens || options.targetContextTokens} token context target`,
+      options.kvOffload ? 'KV offload present' : 'KV offload missing',
+      drop !== null ? `${drop}% throughput drop` : null,
+    ].filter(Boolean),
+    risk: 'Long-context serving can hit memory ceilings or hidden throughput regressions without capacity and benchmark gates.',
+  };
+}
+
+function verifiedReplaySignal(options) {
+  if (!options.training) return null;
+  return {
+    id: 'verified_rl_replay',
+    label: 'Verified RL replay safety',
+    values: [
+      options.rolloutReplay ? 'rollout replay present' : 'rollout replay missing',
+      options.indexerReplay ? 'indexer replay present' : 'indexer replay missing',
+      options.trainInferenceDrift !== null ? `${options.trainInferenceDrift} train-inference drift` : null,
+    ].filter(Boolean),
+    risk: 'Sparse routing and indexer decisions must be replayed or training can optimize against a different path than rollout served.',
+  };
+}
+
+function mixedPrecisionSignal(options) {
+  if (!(usesMixedPrecision(options) || options.numericalSpikes)) return null;
+  return {
+    id: 'mixed_precision_determinism',
+    label: 'Mixed precision determinism',
+    values: [
+      options.precisionMode || 'precision mode unspecified',
+      options.deterministic ? 'determinism enabled' : 'determinism missing',
+      options.numericalSpikes ? 'numerical spikes observed' : null,
+    ].filter(Boolean),
+    risk: 'FP4/FP8 rollout and training can introduce silent numerical drift without deterministic and FP32-sensitive-path checks.',
+  };
+}
+
+function buildDeepSeekV4RuntimeGuardrailsPlan(rawOptions = {}, templatesPath) {
+  const options = normalizeOptions(rawOptions);
+  const templates = listGateTemplates(templatesPath)
+    .filter((template) => template.category === CATEGORY)
+    .map((template) => ({
+      ...template,
+      recommended: templateApplicability(template, options),
+    }));
+  const signals = buildSignals(options);
+  const recommendedTemplates = templates.filter((template) => template.recommended);
+
+  return {
+    name: 'thumbgate-deepseek-v4-runtime-guardrails',
+    status: recommendedTemplates.length > 0 ? 'actionable' : 'ready',
+    workload: options.workload,
+    model: options.model,
+    engine: options.engine,
+    metrics: {
+      contextTokens: options.contextTokens,
+      targetContextTokens: options.targetContextTokens,
+      baselineThroughput: options.baselineThroughput,
+      newThroughput: options.newThroughput,
+      throughputDropPercent: throughputDropPercent(options),
+      acceptLength: options.acceptLength,
+      trainInferenceDrift: options.trainInferenceDrift,
+    },
+    summary: {
+      signalCount: signals.length,
+      templateCount: templates.length,
+      recommendedTemplateCount: recommendedTemplates.length,
+    },
+    signals,
+    templates,
+    nextActions: [
+      'Benchmark DeepSeek-V4 behind the same ThumbGate eval harness before changing routing defaults.',
+      'Require cache-coherence and rollback evidence before enabling hybrid prefix caching or speculative decoding.',
+      'Keep long-context memory and throughput budgets explicit before raising context windows.',
+      'For RL or fine-tuning, require rollout-routing replay, indexer replay, and train-inference drift checks.',
+      'Treat FP4/FP8 or mixed-precision paths as gated rollouts until deterministic and sensitive-FP32 checks pass.',
+    ],
+    exampleCommand: 'npx thumbgate deepseek-v4-runtime-guardrails --context-tokens=900000 --hybrid-attention --speculative-decoding --accept-length=1.4 --precision-mode=fp8 --training --json',
+  };
+}
+
+function formatDeepSeekV4RuntimeGuardrailsPlan(report) {
+  const lines = [
+    '',
+    'ThumbGate DeepSeek-V4 Runtime Guardrails',
+    '-'.repeat(43),
+    `Status  : ${report.status}`,
+    `Workload: ${report.workload}`,
+    `Model   : ${report.model}`,
+    `Engine  : ${report.engine}`,
+    `Signals : ${report.summary.signalCount}`,
+    `Templates: ${report.summary.recommendedTemplateCount}/${report.summary.templateCount} recommended`,
+  ];
+  if (report.metrics.contextTokens !== null) lines.push(`Context tokens: ${report.metrics.contextTokens}`);
+  if (report.metrics.throughputDropPercent !== null) lines.push(`Throughput drop: ${report.metrics.throughputDropPercent}%`);
+  if (report.metrics.acceptLength !== null) lines.push(`Spec accept length: ${report.metrics.acceptLength}`);
+  if (report.metrics.trainInferenceDrift !== null) lines.push(`Train/inference drift: ${report.metrics.trainInferenceDrift}`);
+
+  if (report.signals.length > 0) {
+    lines.push('', 'Detected runtime signals:');
+    for (const signal of report.signals) {
+      lines.push(`  - ${signal.label}: ${signal.values.join(', ')}`);
+      lines.push(`    Risk: ${signal.risk}`);
+    }
+  }
+
+  lines.push('', 'Recommended templates:');
+  const recommended = report.templates.filter((template) => template.recommended);
+  if (recommended.length === 0) lines.push('  - No sparse-attention runtime risks were passed.');
+  for (const template of recommended) {
+    lines.push(`  - ${template.id} [${template.defaultAction}]`);
+    lines.push(`    ${template.roi}`);
+  }
+
+  lines.push('', 'Next actions:');
+  for (const action of report.nextActions) lines.push(`  - ${action}`);
+  lines.push('', `Example: ${report.exampleCommand}`, '');
+  return `${lines.join('\n')}\n`;
+}
+
+module.exports = {
+  buildDeepSeekV4RuntimeGuardrailsPlan,
+  formatDeepSeekV4RuntimeGuardrailsPlan,
+  normalizeOptions,
+  throughputDropPercent,
+};

package/scripts/gemini-embedding-policy.js

@@ -0,0 +1,198 @@
+#!/usr/bin/env node
+'use strict';
+
+const GEMINI_EMBEDDING_2_MODEL = 'gemini-embedding-2';
+const DEFAULT_OUTPUT_DIMENSIONALITY = 768;
+const RECOMMENDED_OUTPUT_DIMENSIONS = [3072, 1536, 768];
+
+const MULTIMODAL_LIMITS = Object.freeze({
+  maxTextTokens: 8192,
+  maxImages: 6,
+  maxVideoSeconds: 120,
+  maxAudioSeconds: 180,
+  maxPdfPages: 6,
+  languages: '100+',
+});
+
+const ASYMMETRIC_TASKS = new Set([
+  'question answering',
+  'fact checking',
+  'code retrieval',
+  'search result',
+]);
+
+const SYMMETRIC_TASKS = new Set([
+  'anomaly detection',
+  'classification',
+  'clustering',
+  'sentence similarity',
+]);
+
+const GEMINI_TASK_TYPES = Object.freeze({
+  query: 'RETRIEVAL_QUERY',
+  document: 'RETRIEVAL_DOCUMENT',
+  classification: 'CLASSIFICATION',
+  clustering: 'CLUSTERING',
+  sentenceSimilarity: 'SEMANTIC_SIMILARITY',
+});
+
+function normalizeTask(task, fallback = 'code retrieval') {
+  const normalized = String(task || fallback)
+    .trim()
+    .toLowerCase()
+    .replace(/[_-]+/g, ' ')
+    .replace(/\s+/g, ' ');
+  return normalized || fallback;
+}
+
+function normalizeEmbeddingKind(kind) {
+  const normalized = String(kind || 'document').trim().toLowerCase();
+  if (normalized === 'query' || normalized === 'document' || normalized === 'symmetric') {
+    return normalized;
+  }
+  return 'document';
+}
+
+function isSymmetricTask(task) {
+  return SYMMETRIC_TASKS.has(normalizeTask(task));
+}
+
+function prepareEmbeddingText({ content, kind = 'document', task = 'code retrieval', title = 'none' } = {}) {
+  const text = String(content || '').trim();
+  const normalizedTask = normalizeTask(task);
+  const normalizedKind = isSymmetricTask(normalizedTask) ? 'symmetric' : normalizeEmbeddingKind(kind);
+
+  if (!text) return '';
+
+  if (normalizedKind === 'query' || normalizedKind === 'symmetric') {
+    return `task: ${normalizedTask} | query: ${text}`;
+  }
+
+  const safeTitle = String(title || 'none').trim() || 'none';
+  return `title: ${safeTitle} | text: ${text}`;
+}
+
+function resolveGeminiTaskType({ kind = 'document', task = 'code retrieval' } = {}) {
+  const normalizedTask = normalizeTask(task);
+  if (normalizedTask === 'classification') return GEMINI_TASK_TYPES.classification;
+  if (normalizedTask === 'clustering') return GEMINI_TASK_TYPES.clustering;
+  if (normalizedTask === 'sentence similarity') return GEMINI_TASK_TYPES.sentenceSimilarity;
+
+  const normalizedKind = normalizeEmbeddingKind(kind);
+  if (normalizedKind === 'query') return GEMINI_TASK_TYPES.query;
+  if (normalizedKind === 'document') return GEMINI_TASK_TYPES.document;
+  return undefined;
+}
+
+function resolveGeminiModelResource(model) {
+  const normalized = String(model || GEMINI_EMBEDDING_2_MODEL).trim() || GEMINI_EMBEDDING_2_MODEL;
+  return normalized.startsWith('models/') ? normalized : `models/${normalized}`;
+}
+
+function parseBoolean(value, fallback = false) {
+  if (value == null || value === '') return fallback;
+  if (typeof value === 'boolean') return value;
+  const normalized = String(value).trim().toLowerCase();
+  if (['1', 'true', 'yes', 'on'].includes(normalized)) return true;
+  if (['0', 'false', 'no', 'off'].includes(normalized)) return false;
+  return fallback;
+}
+
+function parsePositiveInteger(value, fallback) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
+  return Math.floor(parsed);
+}
+
+function normalizeOutputDimensionality(value) {
+  const parsed = parsePositiveInteger(value, DEFAULT_OUTPUT_DIMENSIONALITY);
+  if (RECOMMENDED_OUTPUT_DIMENSIONS.includes(parsed)) return parsed;
+  return RECOMMENDED_OUTPUT_DIMENSIONS.reduce((best, candidate) => (
+    Math.abs(candidate - parsed) < Math.abs(best - parsed) ? candidate : best
+  ), DEFAULT_OUTPUT_DIMENSIONALITY);
+}
+
+function resolveGeminiEmbeddingConfig(env = process.env) {
+  const provider = String(env.THUMBGATE_EMBED_PROVIDER || env.THUMBGATE_EMBEDDING_PROVIDER || 'local')
+    .trim()
+    .toLowerCase();
+  const explicitGemini = parseBoolean(env.THUMBGATE_GEMINI_EMBEDDINGS, false);
+  const apiKey = env.GEMINI_API_KEY || env.GOOGLE_API_KEY || env.GOOGLE_GENERATIVE_AI_API_KEY || '';
+  const enabled = explicitGemini || provider === 'gemini';
+
+  return {
+    enabled,
+    provider: enabled ? 'gemini' : 'local',
+    model: String(env.THUMBGATE_GEMINI_EMBED_MODEL || GEMINI_EMBEDDING_2_MODEL).trim() || GEMINI_EMBEDDING_2_MODEL,
+    apiKey,
+    apiBaseUrl: trimTrailingSlashes(env.THUMBGATE_GEMINI_API_BASE_URL || 'https://generativelanguage.googleapis.com/v1beta'),
+    outputDimensionality: normalizeOutputDimensionality(env.THUMBGATE_GEMINI_EMBED_DIM || env.THUMBGATE_EMBED_DIM),
+    fallbackToLocal: parseBoolean(env.THUMBGATE_GEMINI_EMBED_FALLBACK_LOCAL, true),
+    defaultTask: normalizeTask(env.THUMBGATE_GEMINI_EMBED_TASK || 'code retrieval'),
+    multimodalLimits: MULTIMODAL_LIMITS,
+    recommendedOutputDimensions: RECOMMENDED_OUTPUT_DIMENSIONS,
+  };
+}
+
+function trimTrailingSlashes(value) {
+  let text = String(value);
+  while (text.endsWith('/')) text = text.slice(0, -1);
+  return text;
+}
+
+function buildGeminiEmbeddingRolloutPlan(args = {}) {
+  const corpusItems = parsePositiveInteger(args.corpusItems, 5000);
+  const outputDimensionality = normalizeOutputDimensionality(args.outputDimensionality || args.maxEmbeddingDim);
+  const task = normalizeTask(args.task || 'code retrieval');
+  const useBatchApi = args.useBatchApi !== false;
+  const vectorMb = Number(((corpusItems * outputDimensionality * 4) / (1024 * 1024)).toFixed(2));
+
+  return {
+    model: GEMINI_EMBEDDING_2_MODEL,
+    task,
+    outputDimensionality,
+    corpusItems,
+    estimatedFloat32Mb: vectorMb,
+    taskPrefixes: {
+      query: prepareEmbeddingText({ kind: 'query', task, content: '{query}' }),
+      document: prepareEmbeddingText({ kind: 'document', task, title: '{title}', content: '{content}' }),
+      symmetric: prepareEmbeddingText({ kind: 'symmetric', task: 'classification', content: '{content}' }),
+    },
+    apiHints: {
+      queryTaskType: resolveGeminiTaskType({ kind: 'query', task }),
+      documentTaskType: resolveGeminiTaskType({ kind: 'document', task }),
+      modelResource: resolveGeminiModelResource(GEMINI_EMBEDDING_2_MODEL),
+    },
+    modalityLimits: MULTIMODAL_LIMITS,
+    economics: {
+      recommendedDimensions: RECOMMENDED_OUTPUT_DIMENSIONS,
+      storageDefault: outputDimensionality,
+      batchApi: useBatchApi ? 'Use for offline re-indexing; Google positions Batch API embeddings at 50% of default price.' : 'Skip Batch API only for latency-sensitive incremental writes.',
+    },
+    rolloutSteps: [
+      'Keep local embeddings as the default offline path.',
+      'Enable Gemini Embedding 2 only when a Gemini API key is present.',
+      'Use task-specific query/document prefixes at index and retrieval time.',
+      'Start at 768 dimensions, then benchmark 1536 only if recall misses show up.',
+      'Use Batch API for full re-indexes and online embed_content for fresh feedback events.',
+    ],
+  };
+}
+
+module.exports = {
+  ASYMMETRIC_TASKS,
+  DEFAULT_OUTPUT_DIMENSIONALITY,
+  GEMINI_EMBEDDING_2_MODEL,
+  MULTIMODAL_LIMITS,
+  RECOMMENDED_OUTPUT_DIMENSIONS,
+  SYMMETRIC_TASKS,
+  GEMINI_TASK_TYPES,
+  buildGeminiEmbeddingRolloutPlan,
+  isSymmetricTask,
+  normalizeOutputDimensionality,
+  normalizeTask,
+  prepareEmbeddingText,
+  resolveGeminiEmbeddingConfig,
+  resolveGeminiModelResource,
+  resolveGeminiTaskType,
+};

package/scripts/inference-cache-policy.js

@@ -66,7 +66,46 @@ function evaluateCacheCandidate(candidate = {}) {
   };
 }
 
+function planDepthWiseKvSharing(options = {}) {
+  const layerCount = Number(options.layerCount || 0);
+  const cacheBudgetRatio = Number(options.cacheBudgetRatio || 1);
+  const trainingAdapted = Boolean(options.trainingAdapted || options.randomCrossLayerAttention);
+  const latencySensitive = Boolean(options.latencySensitive);
+  const unknownHardware = Boolean(options.unknownHardware);
+  const dataConstrained = Boolean(options.dataConstrained);
+  const issues = [];
+
+  if (layerCount < 12) issues.push('model_too_shallow_for_depth_sharing_roi');
+  if (!trainingAdapted) issues.push('requires_training_or_finetune_adaptation');
+  if (cacheBudgetRatio >= 0.9) issues.push('kv_memory_budget_not_constrained');
+  if (latencySensitive && !trainingAdapted) issues.push('avoid_runtime_only_cross_layer_sharing_for_ttfb');
+
+  const targetSharedLayerRatio = cacheBudgetRatio <= 0.5 ? 0.5 : cacheBudgetRatio <= 0.75 ? 0.25 : 0;
+  const estimatedKvMemoryReduction = Number((targetSharedLayerRatio * 0.9).toFixed(2));
+  const deploymentModes = [
+    'full-kv-cache',
+    targetSharedLayerRatio >= 0.25 ? 'share-every-fourth-layer' : null,
+    targetSharedLayerRatio >= 0.5 ? 'share-every-other-layer' : null,
+  ].filter(Boolean);
+
+  return {
+    decision: issues.some((issue) => issue !== 'kv_memory_budget_not_constrained') ? 'research' : 'pilot',
+    issues,
+    technique: 'stochastic-kv-routing-depth-wise-cache-sharing',
+    targetSharedLayerRatio,
+    estimatedKvMemoryReduction,
+    deploymentModes,
+    recommendedWorkload: unknownHardware || dataConstrained ? 'adaptive-serving-pilot' : 'benchmark-before-rollout',
+    gates: [
+      'compare quality against full-cache baseline',
+      'measure time-to-first-token and tokens/sec',
+      'block rollout if golden eval pass rate regresses',
+    ],
+  };
+}
+
 module.exports = {
   buildInferenceCachePolicy,
   evaluateCacheCandidate,
+  planDepthWiseKvSharing,
 };