sentinelayer-cli 0.37.1 → 0.38.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sentinelayer-cli",
-  "version": "0.37.1",
+  "version": "0.38.0",
   "description": "Scaffold Sentinelayer spec/prompt/guide artifacts with secure browser auth and token bootstrap.",
   "type": "module",
   "scripts": {
@@ -48,6 +48,7 @@
     "url": "https://github.com/mrrCarter/create-sentinelayer/issues"
   },
   "dependencies": {
+    "@anthropic-ai/tokenizer": "0.0.4",
     "@axe-core/playwright": "4.11.2",
     "@babel/parser": "7.29.2",
     "axe-core": "4.11.3",
@@ -60,6 +61,7 @@
     "picocolors": "1.1.1",
     "playwright": "1.59.1",
     "prompts": "2.4.2",
+    "tiktoken": "1.0.22",
     "yaml": "2.8.3",
     "zod": "4.1.12"
   },

package/src/cost/tokenizer.js

@@ -1,21 +1,24 @@
+import { countTokens as countAnthropicTokens } from "@anthropic-ai/tokenizer";
+import { encoding_for_model, get_encoding } from "tiktoken";
+
 // Provider-aware token estimator (#A12, spec §5.2).
 //
 // The rest of the CLI has been guessing token counts with `text.length / 4`
 // since v0.1. That's off by 20-40% vs. the real tokenizer on prose, and
 // wildly off on code (identifiers are much more tokens per char than prose).
-// This module ships a zero-dep heuristic that is significantly more accurate
-// and — critically — provider-aware so budget calculations stop rewarding
-// whoever has the larger BPE vocabulary.
+// This module uses the real Anthropic and OpenAI tokenizers where available
+// and keeps the calibrated provider-aware heuristic as the fail-safe for
+// unsupported providers or tokenizer runtime failures.
 //
 // Design goals:
-//   - Zero runtime dependencies. @anthropic-ai/tokenizer and tiktoken are
-//     multi-MB WASM payloads we're not willing to add at CLI-install time.
-//   - API stable enough that swapping in the real tokenizer later is a
-//     strict drop-in — pass `{ backend: fn }` to `estimateTokens` and the
-//     backend takes precedence over the heuristic.
+//   - Real tokenizer first for supported providers.
+//   - API stable enough for tests and callers that need custom counting:
+//     pass `{ backend: fn }` to `estimateTokens` and the backend takes
+//     precedence over the built-in tokenizers.
 //   - Calibrated ratios per provider family. Numbers below are measured
 //     against published BPE stats for cl100k_base (OpenAI), claude (Anthropic),
-//     and gemini (Google) across a mix of English prose + JS/TS source.
+//     and gemini (Google) across a mix of English prose + JS/TS source, and
+//     are only used when a real tokenizer cannot be applied.
 
 const PROVIDER_FAMILIES = Object.freeze(["anthropic", "openai", "google", "unknown"]);
 
@@ -51,6 +54,8 @@ const MODEL_PROVIDER_RULES = [
   { pattern: /^google[/:]/i, family: "google" },
 ];
 
+const OPENAI_DEFAULT_ENCODING = "cl100k_base";
+
 // Detect provider family from a loose model id: Anthropic conventions like
 // "claude-opus-4-7", OpenAI "gpt-5.3-codex" / "o4-mini" / "codex-mini-2026",
 // Google "gemini-2.5-pro". Unknown ids fall back to the generic tokenizer.
@@ -85,50 +90,111 @@ function countWords(text) {
   return parts.length;
 }
 
+function safePositiveCeil(value) {
+  const normalized = Number(value);
+  if (!Number.isFinite(normalized) || normalized < 0) {
+    return null;
+  }
+  return Math.max(1, Math.ceil(normalized));
+}
+
+function heuristicEstimateTokens(str, family) {
+  const charsPerToken = CHARS_PER_TOKEN[family] || CHARS_PER_TOKEN.unknown;
+  const tokensPerWord = TOKENS_PER_WORD[family] || TOKENS_PER_WORD.unknown;
+
+  const normalized = str.replace(/\s+/g, " ").trim();
+  if (!normalized) {
+    return 0;
+  }
+  const charEstimate = Math.ceil(normalized.length / charsPerToken);
+  const wordEstimate = Math.ceil(countWords(normalized) * tokensPerWord);
+  // Blend: the higher-accuracy answer depends on whether the input is
+  // whitespace-sparse (code/json/base64 — char estimate wins) or
+  // whitespace-dense prose (word estimate is more accurate). Take the max
+  // of the two, because underestimating token counts blows budgets; this
+  // biases cost estimates slightly on the safe side.
+  return Math.max(1, charEstimate, wordEstimate);
+}
+
+function countOpenAiTokens(str, model) {
+  let encoder = null;
+  try {
+    encoder = model
+      ? encoding_for_model(String(model).trim())
+      : get_encoding(OPENAI_DEFAULT_ENCODING);
+  } catch {
+    encoder = get_encoding(OPENAI_DEFAULT_ENCODING);
+  }
+
+  try {
+    return encoder.encode(str).length;
+  } finally {
+    if (encoder && typeof encoder.free === "function") {
+      encoder.free();
+    }
+  }
+}
+
+export function countWithProviderTokenizer(
+  text,
+  { provider = "", model = "" } = {}
+) {
+  const str = typeof text === "string" ? text : text == null ? "" : String(text);
+  if (!str || !str.trim()) {
+    return 0;
+  }
+
+  let family = normalizeProviderFamily(provider);
+  if (family === "unknown" && model) {
+    family = detectProviderFamily(model);
+  }
+
+  if (family === "anthropic") {
+    return safePositiveCeil(countAnthropicTokens(str));
+  }
+  if (family === "openai") {
+    return safePositiveCeil(countOpenAiTokens(str, model));
+  }
+  return null;
+}
+
 // Estimate token count for a text against a provider family. Uses a blend
-// of char-per-token and word-per-token so short inputs (which are mostly
-// function of token-per-word behavior) and long runs of no-break chars
-// (where the char ratio dominates) both get sensible answers.
+// of real provider tokenizers and the provider-aware fallback heuristic.
 //
 // Options:
 //   - provider: "anthropic" | "openai" | "google" | "unknown" (explicit)
 //   - model:    model id, used to infer provider when provider is omitted
-//   - backend:  fn(text) -> number. Overrides the heuristic. This is the
-//               hook for swapping in @anthropic-ai/tokenizer / tiktoken
-//               without rewriting callers.
+//   - backend:  fn(text) -> number. Overrides the built-in tokenizer path.
 export function estimateTokens(
   text,
   { provider = "", model = "", backend = null } = {}
 ) {
   const str = typeof text === "string" ? text : text == null ? "" : String(text);
-  if (!str) {
+  if (!str || !str.trim()) {
     return 0;
   }
   if (typeof backend === "function") {
-    const custom = Number(backend(str));
-    if (Number.isFinite(custom) && custom >= 0) {
-      return Math.max(1, Math.ceil(custom));
+    const custom = safePositiveCeil(backend(str));
+    if (custom !== null) {
+      return custom;
     }
   }
+
   let family = normalizeProviderFamily(provider);
   if (family === "unknown" && model) {
     family = detectProviderFamily(model);
   }
-  const charsPerToken = CHARS_PER_TOKEN[family] || CHARS_PER_TOKEN.unknown;
-  const tokensPerWord = TOKENS_PER_WORD[family] || TOKENS_PER_WORD.unknown;
 
-  const normalized = str.replace(/\s+/g, " ").trim();
-  if (!normalized) {
-    return 0;
+  try {
+    const tokenizerCount = countWithProviderTokenizer(str, { provider: family, model });
+    if (tokenizerCount !== null) {
+      return tokenizerCount;
+    }
+  } catch {
+    // Keep budget calculations available even if a native/WASM tokenizer
+    // cannot initialize in a constrained runtime.
   }
-  const charEstimate = Math.ceil(normalized.length / charsPerToken);
-  const wordEstimate = Math.ceil(countWords(normalized) * tokensPerWord);
-  // Blend: the higher-accuracy answer depends on whether the input is
-  // whitespace-sparse (code/json/base64 — char estimate wins) or
-  // whitespace-dense prose (word estimate is more accurate). Take the max
-  // of the two, because underestimating token counts blows budgets; this
-  // biases cost estimates slightly on the safe side.
-  return Math.max(1, charEstimate, wordEstimate);
+  return heuristicEstimateTokens(str, family);
 }
 
 // Combined token count + cost calculation for a single request. Consumers
@@ -157,4 +223,9 @@ export function estimateTokensForMessages(
   return total;
 }
 
-export { CHARS_PER_TOKEN, PROVIDER_FAMILIES, TOKENS_PER_WORD };
+export {
+  CHARS_PER_TOKEN,
+  OPENAI_DEFAULT_ENCODING,
+  PROVIDER_FAMILIES,
+  TOKENS_PER_WORD,
+};

package/src/legacy-cli.js

@@ -2517,7 +2517,7 @@ jobs:
           fi
       - name: Run Omar Gate
         id: omar
-        uses: mrrCarter/sentinelayer-v1-action@5c4d8c175eb117ea5256452e50e01249ab126998
+        uses: mrrCarter/sentinelayer-v1-action@a496be33a466c0cc3f8616d66bbd7d78f7d3c31d
         with:
           github_token: \${{ github.token }}
           openai_api_key: \${{ secrets.OPENAI_API_KEY }}
@@ -2527,9 +2527,9 @@ jobs:
           sentinelayer_managed_llm: \${{ secrets.${normalizedSecret} != '' }}
           scan_mode: \${{ github.event_name == 'workflow_dispatch' && inputs.scan_mode || 'deep' }}
           severity_gate: \${{ github.event_name == 'workflow_dispatch' && inputs.severity_gate || 'P1' }}
-          model: \${{ secrets.OPENAI_API_KEY != '' && 'gpt-5.3-codex' || ((secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-3.1-flash-lite' || 'gpt-5.3-codex') }}
+          model: \${{ secrets.OPENAI_API_KEY != '' && 'gpt-5.3-codex' || ((secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-2.5-flash' || 'gpt-5.3-codex') }}
           codex_model: gpt-5.3-codex
-          model_fallback: \${{ (secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-3.1-flash-lite' || 'gpt-4.1-mini' }}
+          model_fallback: \${{ (secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-2.5-flash' || 'gpt-4.1-mini' }}
           use_codex: \${{ secrets.OPENAI_API_KEY != '' || (secrets.GOOGLE_GEMINI_API_KEY == '' && secrets.GOOGLE_API_KEY == '') }}
           codex_only: "false"
           llm_failure_policy: block
@@ -2585,8 +2585,8 @@ jobs:
           OMAR_GOOGLE_KEY_PRESENT: \${{ secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '' }}
           OMAR_MANAGED_LLM: \${{ secrets.${normalizedSecret} != '' }}
           OMAR_LLM_PROVIDER: \${{ secrets.OPENAI_API_KEY != '' && 'openai' || ((secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'google' || 'openai') }}
-          OMAR_MODEL: \${{ secrets.OPENAI_API_KEY != '' && 'gpt-5.3-codex' || ((secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-3.1-flash-lite' || 'gpt-5.3-codex') }}
-          OMAR_MODEL_FALLBACK: \${{ (secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-3.1-flash-lite' || 'gpt-4.1-mini' }}
+          OMAR_MODEL: \${{ secrets.OPENAI_API_KEY != '' && 'gpt-5.3-codex' || ((secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-2.5-flash' || 'gpt-5.3-codex') }}
+          OMAR_MODEL_FALLBACK: \${{ (secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-2.5-flash' || 'gpt-4.1-mini' }}
         run: |
           set -euo pipefail
           mkdir -p omar-artifacts
@@ -2626,7 +2626,7 @@ jobs:
               },
               "scan": {
                   "mode": env("OMAR_SCAN_MODE", "deep"),
-                  "action_ref": "mrrCarter/sentinelayer-v1-action@5c4d8c175eb117ea5256452e50e01249ab126998",
+                  "action_ref": "mrrCarter/sentinelayer-v1-action@a496be33a466c0cc3f8616d66bbd7d78f7d3c31d",
                   "llm_provider": env("OMAR_LLM_PROVIDER", "openai"),
                   "model": env("OMAR_MODEL", "gpt-5.3-codex"),
                   "model_fallback": env("OMAR_MODEL_FALLBACK", "gpt-4.1-mini"),

package/src/review/omargate-orchestrator.js

@@ -376,6 +376,52 @@ function summarizeFindings(findings = []) {
   };
 }
 
+function buildPersonaHealthFindings(personaHealth = {}) {
+  if (personaHealth.healthy || personaHealth.total <= 0) {
+    return [];
+  }
+
+  const warnings = Array.isArray(personaHealth.warnings) ? personaHealth.warnings : [];
+  const errorCount = Number(personaHealth.error || 0);
+  const total = Number(personaHealth.total || 0);
+  const errorRatio = Number(personaHealth.errorRatio || 0);
+  const totalCostUsd = Number(personaHealth.totalCostUsd || 0);
+  const allPersonasFailed = total > 0 && errorCount === total;
+  const severity = allPersonasFailed ? "P0" : "P1";
+  let message = `Omar AI coverage degraded: ${errorCount}/${total} dispatched personas errored (${Math.round(errorRatio * 100)}%).`;
+  if (allPersonasFailed) {
+    message = `Omar AI coverage failed closed: all ${total} dispatched personas errored before producing findings.`;
+  } else if (errorCount === 0 && totalCostUsd <= 0) {
+    message = "Omar AI coverage failed closed: personas reported ok but total AI cost was $0.00, indicating no LLM calls were made.";
+  }
+
+  return [
+    {
+      severity,
+      file: "<omargate>",
+      line: 1,
+      message,
+      category: "Omar AI Coverage",
+      source: "orchestrator",
+      layer: "orchestrator",
+      confidence: 1,
+      confidenceFloor: 0,
+      sources: ["orchestrator"],
+      remediation:
+        "Fix SentinelLayer managed LLM proxy/provider credentials or run with --no-ai when deterministic-only coverage is intentional.",
+      evidence: warnings,
+      personaHealth: {
+        ok: Number(personaHealth.ok || 0),
+        error: errorCount,
+        skipped: Number(personaHealth.skipped || 0),
+        total,
+        errorRatio,
+        totalCostUsd,
+      },
+    },
+  ];
+}
+
 function personaAgentFromIdentity(identity = {}) {
   return {
     id: identity.id,
@@ -443,6 +489,7 @@ async function runOmarPersonaSwarm({
   usageSessionId = "",
   requireUsageLedger = false,
   usageRecorder = undefined,
+  aiReviewRunner = runAiReviewLayer,
 } = {}) {
   const scope = buildPersonaFileScope({ deterministic });
   const decision = decideSwarm({ scope });
@@ -518,7 +565,7 @@ async function runOmarPersonaSwarm({
       }
 
       try {
-        const result = await runAiReviewLayer({
+        const result = await aiReviewRunner({
           targetPath,
           mode: "full",
           runId: subagentRunId,
@@ -827,6 +874,7 @@ export async function runOmarGateOrchestrator({
   requireUsageLedger = false,
   usageSessionId = "",
   usageRecorder = undefined,
+  aiReviewRunner = runAiReviewLayer,
 } = {}) {
   const runId = `omargate-${Date.now()}-${randomUUID().slice(0, 8)}`;
   const startTime = Date.now();
@@ -999,6 +1047,7 @@ export async function runOmarGateOrchestrator({
         requireUsageLedger,
         usageSessionId,
         usageRecorder,
+        aiReviewRunner,
       });
 
       if (swarmResult) {
@@ -1048,7 +1097,7 @@ export async function runOmarGateOrchestrator({
         deterministicSummary: personaDeterministic?.summary || detSummary,
       });
 
-      const result = await runAiReviewLayer({
+      const result = await aiReviewRunner({
         targetPath,
         mode: "full",
         runId: `${runId}-${personaId}`,
@@ -1210,8 +1259,8 @@ export async function runOmarGateOrchestrator({
     defaultConfidenceFloor: OMARGATE_DEFAULT_CONFIDENCE_FLOOR,
     confidenceFloors: OMARGATE_CONFIDENCE_FLOORS,
   });
-  const reconciledFindings = reconciled.findings;
-  const reconciledSummary = reconciled.summary;
+  let reconciledFindings = reconciled.findings;
+  let reconciledSummary = reconciled.summary;
   const droppedBelowConfidence = Number(reconciledSummary?.droppedBelowConfidence || 0);
   const candidateFindingCount = detFindings.length + allAiFindings.length;
   const dedupedCount = Math.max(
@@ -1241,6 +1290,7 @@ export async function runOmarGateOrchestrator({
     skipped: personaSkippedCount,
     total: totalPersonas,
     errorRatio,
+    totalCostUsd: totalCost,
     healthy: aiCoverageHealthy || dryRun,
     warnings: [],
   };
@@ -1261,6 +1311,12 @@ export async function runOmarGateOrchestrator({
     }
   }
 
+  const personaHealthFindings = buildPersonaHealthFindings(personaHealth);
+  if (personaHealthFindings.length > 0) {
+    reconciledFindings = [...reconciledFindings, ...personaHealthFindings];
+    reconciledSummary = summarizeFindings(reconciledFindings);
+  }
+
   const result = {
     runId,
     mode,
@@ -1286,6 +1342,7 @@ export async function runOmarGateOrchestrator({
     findingsBySource: {
       deterministic: detFindings.length,
       ai: allAiFindings.length,
+      orchestrator: personaHealthFindings.length,
       reconciled: reconciledFindings.length,
       droppedBelowConfidence,
     },
@@ -1295,6 +1352,7 @@ export async function runOmarGateOrchestrator({
     reconciliation: {
       deterministicFindings: detFindings.length,
       aiFindings: allAiFindings.length,
+      orchestratorFindings: personaHealthFindings.length,
       reconciledFindings: reconciledFindings.length,
       dedupedCount,
       droppedBelowConfidence,

package/src/scan/generator.js

@@ -2,7 +2,7 @@ import YAML from "yaml";
 
 export const DEFAULT_SCAN_WORKFLOW_PATH = ".github/workflows/omar-gate.yml";
 export const DEFAULT_SCAN_SECRET_NAME = "SENTINELAYER_TOKEN";
-export const SENTINELAYER_ACTION_REF = "mrrCarter/sentinelayer-v1-action@5c4d8c175eb117ea5256452e50e01249ab126998";
+export const SENTINELAYER_ACTION_REF = "mrrCarter/sentinelayer-v1-action@a496be33a466c0cc3f8616d66bbd7d78f7d3c31d";
 export const SUPPORTED_E2E_HINTS = Object.freeze(["auto", "yes", "no"]);
 export const SUPPORTED_PLAYWRIGHT_MODES = Object.freeze(["auto", "off", "baseline", "audit"]);
 
@@ -236,10 +236,10 @@ export function buildSecurityReviewWorkflow({ secretName = DEFAULT_SCAN_SECRET_N
               scan_mode: profile.scanMode || "deep",
               severity_gate: profile.severityGate || "P1",
               model:
-                "${{ secrets.OPENAI_API_KEY != '' && 'gpt-5.3-codex' || ((secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-3.1-flash-lite' || 'gpt-5.3-codex') }}",
+                "${{ secrets.OPENAI_API_KEY != '' && 'gpt-5.3-codex' || ((secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-2.5-flash' || 'gpt-5.3-codex') }}",
               codex_model: "gpt-5.3-codex",
               model_fallback:
-                "${{ (secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-3.1-flash-lite' || 'gpt-4.1-mini' }}",
+                "${{ (secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-2.5-flash' || 'gpt-4.1-mini' }}",
               use_codex:
                 "${{ secrets.OPENAI_API_KEY != '' || (secrets.GOOGLE_GEMINI_API_KEY == '' && secrets.GOOGLE_API_KEY == '') }}",
               codex_only: "false",
@@ -306,9 +306,9 @@ export function buildSecurityReviewWorkflow({ secretName = DEFAULT_SCAN_SECRET_N
               OMAR_LLM_PROVIDER:
                 "${{ secrets.OPENAI_API_KEY != '' && 'openai' || ((secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'google' || 'openai') }}",
               OMAR_MODEL:
-                "${{ secrets.OPENAI_API_KEY != '' && 'gpt-5.3-codex' || ((secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-3.1-flash-lite' || 'gpt-5.3-codex') }}",
+                "${{ secrets.OPENAI_API_KEY != '' && 'gpt-5.3-codex' || ((secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-2.5-flash' || 'gpt-5.3-codex') }}",
               OMAR_MODEL_FALLBACK:
-                "${{ (secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-3.1-flash-lite' || 'gpt-4.1-mini' }}",
+                "${{ (secrets.GOOGLE_GEMINI_API_KEY != '' || secrets.GOOGLE_API_KEY != '') && 'gemini-2.5-flash' || 'gpt-4.1-mini' }}",
             },
             run: [
               "set -euo pipefail",