sandoichi 0.4.1 → 0.5.0

package/src/telemetry.mjs

@@ -5,16 +5,44 @@ import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 import { atomicWrite, ensureDirectory, withLock } from './provider-usage.mjs';
+import { PLUGIN_VERSION } from './version.mjs';
 
-const SCHEMA_VERSION = 1;
+export const SCHEMA_VERSION = 2;
+const LEGACY_SCHEMA_VERSION = 1;
+const SUPPORTED_QUEUE_SCHEMA_VERSIONS = new Set([LEGACY_SCHEMA_VERSION, SCHEMA_VERSION]);
 const MAX_STRING_LENGTH = 32;
 const MAX_EVENT_BYTES = 2048;
 
-const COUNT_BUCKETS = ['zero', 'one', '2_to_5', '6_to_20', 'gt_20'];
-const BYTE_BUCKETS = ['lt_4k', '4_to_16k', '16_to_64k', 'gte_64k'];
+const COUNT_BUCKETS = ['zero', 'one', '2_to_5', '6_to_20', '21_to_100', 'gt_100'];
+const BYTE_BUCKETS = ['lt_4k', '4_to_16k', '16_to_64k', '64_to_256k', '256k_to_1m', 'gte_1m'];
+const LOCAL_ONLY_EVENTS = new Set(['f1_footprint', 'f4_gateway']);
 const HOSTS = ['claude', 'codex'];
 const PROVIDERS = ['anthropic', 'openai', 'unknown'];
 const MODES = ['enforce', 'observe', 'dry_run'];
+const F4_HOSTS = ['claude', 'codex', 'unknown'];
+const F4_OPERATIONS = ['catalog', 'call'];
+const F4_OUTCOMES = ['success', 'rejected', 'timeout', 'cancelled', 'error'];
+const F4_LATENCY_BUCKETS = ['lt_10ms', '10_to_100ms', '100_to_1000ms', 'gte_1000ms'];
+const F4_RESULT_BUCKETS = ['zero', 'one', '2_to_5', '6_to_20', 'gt_20', 'unknown'];
+const F1_HOSTS = ['claude', 'codex'];
+const F1_STATUSES = ['complete', 'partial', 'unavailable'];
+const F1_RATIO_BUCKETS = ['zero', 'lt_1pct', '1_to_10pct', 'gt_10pct', 'unavailable'];
+const F1_SIZE_BUCKETS = [...BYTE_BUCKETS, 'unavailable'];
+const F1_INPUT_BUCKETS = [...COUNT_BUCKETS, 'unavailable'];
+// Reduction without coverage reads as better than it is: a day that bounds heavily on the 3% of
+// commands it recognises looks identical to one that bounds everything. These are the reasons the
+// shell classifier already emits, so they are a closed set and carry nothing free-form.
+// Counts alone cannot answer "how much did it reach": 1,276 routed and 43,945 bypassed both land
+// in `gt_100`, and so would the reverse. The ratio is the field that carries the answer.
+export const COVERAGE_RATIO_BUCKETS = ['zero', 'lt_1pct', '1_to_10pct', '10_to_50pct', '50_to_90pct', 'gt_90pct'];
+
+export const COVERAGE_REASONS = [
+  'ambiguous-shell', 'compound-feeds-pipeline', 'compound-has-redirect', 'compound-segment-ambiguous',
+  'grep-shape', 'head-shape', 'invalid-input', 'read-shape', 'routing-disabled', 'sed-shape',
+  'tail-unbounded-from-end', 'unsafe-cwd', 'unsafe-grep-pattern', 'unsafe-grep-target',
+  'unsafe-read-target', 'unsupported-shell', 'unsupported-tool', 'other',
+];
+
 export const FAILURE_STAGES = [
   'policy', 'input', 'redaction', 'optimization', 'artifact', 'output', 'upstream', 'response',
 ];
@@ -23,6 +51,7 @@ const SHARED_FIELDS = {
   schema_version: (value) => value === SCHEMA_VERSION,
   event: (value) => [
     'hook_summary', 'proxy_summary', 'active_day', 'hook_failure_summary', 'proxy_failure_summary',
+    'f1_footprint', 'f4_gateway', 'coverage_summary',
   ].includes(value),
   day_utc: (value) => typeof value === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(value),
   plugin_version: (value) => typeof value === 'string' && /^\d+\.\d+(?:\.\d+)?$/.test(value) && value.length <= MAX_STRING_LENGTH,
@@ -54,12 +83,37 @@ const PROXY_FAILURE_FIELDS = {
   provider: (value) => PROVIDERS.includes(value),
   failure_stage: (value) => FAILURE_STAGES.includes(value),
 };
+const COVERAGE_FIELDS = {
+  host: (value) => HOSTS.includes(value),
+  routed_bucket: (value) => COUNT_BUCKETS.includes(value),
+  bypassed_bucket: (value) => COUNT_BUCKETS.includes(value),
+  coverage_ratio_bucket: (value) => COVERAGE_RATIO_BUCKETS.includes(value),
+  top_bypass_reason: (value) => COVERAGE_REASONS.includes(value),
+};
+
+const F4_FIELDS = {
+  f4_host: (value) => F4_HOSTS.includes(value),
+  f4_operation: (value) => F4_OPERATIONS.includes(value),
+  f4_outcome: (value) => F4_OUTCOMES.includes(value),
+  f4_latency_bucket: (value) => F4_LATENCY_BUCKETS.includes(value),
+  f4_result_bucket: (value) => F4_RESULT_BUCKETS.includes(value),
+};
+const F1_FIELDS = {
+  f1_host: (value) => F1_HOSTS.includes(value),
+  f1_status: (value) => F1_STATUSES.includes(value),
+  f1_unknown_ratio_bucket: (value) => F1_RATIO_BUCKETS.includes(value),
+  f1_body_size_bucket: (value) => F1_SIZE_BUCKETS.includes(value),
+  f1_input_tokens_bucket: (value) => F1_INPUT_BUCKETS.includes(value),
+};
 
 function fieldsForEvent(eventType) {
+  if (eventType === 'f1_footprint') return F1_FIELDS;
   if (eventType === 'hook_summary') return HOOK_FIELDS;
   if (eventType === 'proxy_summary') return PROXY_FIELDS;
   if (eventType === 'active_day') return ACTIVE_DAY_FIELDS;
   if (eventType === 'hook_failure_summary') return HOOK_FAILURE_FIELDS;
+  if (eventType === 'f4_gateway') return F4_FIELDS;
+  if (eventType === 'coverage_summary') return COVERAGE_FIELDS;
   return PROXY_FAILURE_FIELDS;
 }
 
@@ -69,7 +123,8 @@ export function countBucket(count) {
   if (count === 1) return 'one';
   if (count <= 5) return '2_to_5';
   if (count <= 20) return '6_to_20';
-  return 'gt_20';
+  if (count <= 100) return '21_to_100';
+  return 'gt_100';
 }
 
 export function byteBucket(bytes) {
@@ -77,7 +132,9 @@ export function byteBucket(bytes) {
   if (bytes < 4096) return 'lt_4k';
   if (bytes < 16384) return '4_to_16k';
   if (bytes < 65536) return '16_to_64k';
-  return 'gte_64k';
+  if (bytes < 262144) return '64_to_256k';
+  if (bytes < 1048576) return '256k_to_1m';
+  return 'gte_1m';
 }
 
 export function validateEvent(payload) {
@@ -106,11 +163,6 @@ export const TELEMETRY_CONFIG_VERSION = 1;
 export const CONSENT_VERSION = 1;
 export const TELEMETRY_DETAILS_URL = 'https://github.com/yuzushi-dev/Sando/blob/main/TELEMETRY.md';
 export const CONSENT_STATES = ['unasked', 'asked', 'enabled', 'declined'];
-// Canary phase: shared backend, fronted by a Cloudflare Tunnel so it's
-// reachable from any of the owner's machines (see
-// session-handoff/deploy/telemetry/). Rate-limited at nginx (30 req/min/IP).
-// Release/broader publication is still gated on the open items in
-// session-handoff/docs/telemetry-canary-report.md.
 export const TELEMETRY_ENDPOINT = 'https://telemetry.yuzushi.party/v1/logs';
 
 export function isDoNotTrack(env = process.env) {
@@ -230,16 +282,26 @@ const LEASE_MS = 5 * 60 * 1000;
 const RETRY_DELAYS_MS = [60_000, 300_000, 1_800_000, 7_200_000, 21_600_000];
 const CHILD_ENV_KEYS = ['HTTPS_PROXY', 'HTTP_PROXY', 'NO_PROXY', 'NODE_EXTRA_CA_CERTS', 'SSL_CERT_FILE'];
 
-function emptyCounters() { return { schema_version: TELEMETRY_CONFIG_VERSION, counters: {}, active_days: {} }; }
+function emptyCounters() { return { schema_version: TELEMETRY_CONFIG_VERSION, counters: {}, active_days: {}, coverage_days: {} }; }
 function readCounters(countersPath) {
   if (!fs.existsSync(countersPath)) return emptyCounters();
   const state = JSON.parse(fs.readFileSync(countersPath, 'utf8'));
-  return { ...state, counters: state.counters ?? {}, active_days: state.active_days ?? {} };
+  return { ...state, counters: state.counters ?? {}, active_days: state.active_days ?? {}, coverage_days: state.coverage_days ?? {} };
 }
 
 function readQueueRows(queuePath) {
   if (!fs.existsSync(queuePath)) return [];
-  return fs.readFileSync(queuePath, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line));
+  return fs.readFileSync(queuePath, 'utf8').split('\n').filter(Boolean).map((line) => {
+    const row = JSON.parse(line);
+    assertQueueRow(row);
+    return row;
+  });
+}
+
+function assertQueueRow(row) {
+  if (!record(row) || !SUPPORTED_QUEUE_SCHEMA_VERSIONS.has(row.schema_version)) {
+    throw new Error('telemetry queue event schema is unsupported');
+  }
 }
 
 function writeQueueRows(queuePath, rows) {
@@ -253,7 +315,11 @@ function writeQueueRows(queuePath, rows) {
 
 /** Enforces the bounded queue (4096 rows / 4 MiB), dropping the oldest rows first —
  *  a telemetry backlog must never grow without bound or block product behavior. */
-function appendQueueRows(queuePath, newRows) {
+export function appendQueueRows(queuePath, newRows) {
+  for (const row of newRows) {
+    assertQueueRow(row);
+    if (LOCAL_ONLY_EVENTS.has(row.event)) throw new Error(`${row.event}: local-only event must never reach the upload queue`);
+  }
   ensureDirectory(path.dirname(queuePath));
   withLock(`${queuePath}.lock`, () => {
     let rows = readQueueRows(queuePath);
@@ -277,9 +343,10 @@ function publicRow(row) {
 }
 
 function bucketEntry(entry, pluginVersion) {
+  const recordedVersion = entry.pluginVersion ?? pluginVersion;
   if (entry.event === 'hook_summary') {
     return {
-      schema_version: TELEMETRY_CONFIG_VERSION, event: 'hook_summary', day_utc: entry.day, plugin_version: pluginVersion,
+      schema_version: SCHEMA_VERSION, event: 'hook_summary', day_utc: entry.day, plugin_version: recordedVersion,
       host: entry.host, mode: entry.mode,
       tool_calls_bucket: countBucket(entry.toolCalls ?? 0),
       capped_outputs_bucket: countBucket(entry.cappedOutputs ?? 0),
@@ -288,41 +355,70 @@ function bucketEntry(entry, pluginVersion) {
     };
   }
   if (entry.event === 'proxy_summary') return {
-    schema_version: TELEMETRY_CONFIG_VERSION, event: 'proxy_summary', day_utc: entry.day, plugin_version: pluginVersion,
+    schema_version: SCHEMA_VERSION, event: 'proxy_summary', day_utc: entry.day, plugin_version: recordedVersion,
     provider: entry.provider ?? 'unknown', mode: entry.mode ?? 'enforce',
     rewrites_applied_bucket: countBucket(entry.rewritesApplied ?? 0),
     rewrites_skipped_cache_bucket: countBucket(entry.rewritesSkippedCache ?? 0),
     input_tokens_saved_bucket: byteBucket(entry.inputTokensSaved ?? 0),
   };
+  if (entry.event === 'coverage_summary') {
+    const reasons = Object.entries(entry)
+      .filter(([field, count]) => field.startsWith(REASON_PREFIX) && Number.isInteger(count) && count > 0)
+      .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
+    const leading = reasons[0]?.[0].slice(REASON_PREFIX.length);
+    const routed = entry.routed ?? 0;
+    const bypassed = entry.bypassed ?? 0;
+    return {
+      schema_version: SCHEMA_VERSION, event: 'coverage_summary', day_utc: entry.day, plugin_version: recordedVersion,
+      host: entry.host,
+      routed_bucket: countBucket(routed),
+      bypassed_bucket: countBucket(bypassed),
+      coverage_ratio_bucket: coverageRatioBucket(routed, routed + bypassed),
+      // A reason this build does not know travels as `other`, never as free text.
+      top_bypass_reason: COVERAGE_REASONS.includes(leading) ? leading : 'other',
+    };
+  }
   if (entry.event === 'hook_failure_summary') return {
-    schema_version: TELEMETRY_CONFIG_VERSION, event: 'hook_failure_summary', day_utc: entry.day, plugin_version: pluginVersion,
+    schema_version: SCHEMA_VERSION, event: 'hook_failure_summary', day_utc: entry.day, plugin_version: recordedVersion,
     host: entry.host, failure_stage: entry.failureStage,
   };
   return {
-    schema_version: TELEMETRY_CONFIG_VERSION, event: 'proxy_failure_summary', day_utc: entry.day, plugin_version: pluginVersion,
+    schema_version: SCHEMA_VERSION, event: 'proxy_failure_summary', day_utc: entry.day, plugin_version: recordedVersion,
     provider: entry.provider, failure_stage: entry.failureStage,
   };
 }
 
 /** Accumulates raw per-day counts in memory/on disk; values are only bucketed (and thus
  *  only ever leave the machine) once `closeDay` closes a finished UTC day. */
-export function incrementCounter({ statePaths, day, event, host, provider, mode, failureStage, deltas = {} }) {
-  if (!['hook_summary', 'proxy_summary', 'hook_failure_summary', 'proxy_failure_summary'].includes(event)) {
+export function incrementCounter({ statePaths, day, pluginVersion = PLUGIN_VERSION, event, host, provider, mode, failureStage, deltas = {} }) {
+  if (!['hook_summary', 'proxy_summary', 'hook_failure_summary', 'proxy_failure_summary', 'coverage_summary'].includes(event)) {
     throw new Error('incrementCounter: invalid event');
   }
   const isProxy = event.startsWith('proxy_');
   const dimension = isProxy ? provider : host;
-  const key = event.includes('failure')
-    ? [day, event, dimension, failureStage ?? ''].join('|')
-    : [day, event, dimension, mode ?? ''].join('|');
+  if (!SHARED_FIELDS.plugin_version(pluginVersion)) throw new Error('incrementCounter: invalid plugin version');
+  const suffix = event.includes('failure') ? failureStage ?? '' : mode ?? '';
+  const key = [day, pluginVersion, event, dimension, suffix].join('|');
+  const legacyKey = [day, event, dimension, suffix].join('|');
   ensureDirectory(path.dirname(statePaths.counters));
   withLock(`${statePaths.counters}.lock`, () => {
     const state = readCounters(statePaths.counters);
-    const existing = state.counters[key] ?? {
-      day, event, ...(isProxy ? { provider: dimension } : { host: dimension }),
+    const legacy = state.counters[legacyKey];
+    let existing = state.counters[key];
+    if (legacy && !legacy.pluginVersion) {
+      if (existing) {
+        for (const [field, value] of Object.entries(legacy)) {
+          if (Number.isInteger(value) && value >= 0) existing[field] = (existing[field] ?? 0) + value;
+        }
+      } else existing = legacy;
+      delete state.counters[legacyKey];
+    }
+    existing ??= {
+      day, pluginVersion, event, ...(isProxy ? { provider: dimension } : { host: dimension }),
       ...(event.endsWith('_summary') && !event.includes('failure') ? { mode: mode ?? null } : {}),
       ...(event.includes('failure') ? { failureStage } : {}),
     };
+    existing.pluginVersion = pluginVersion;
     for (const [field, value] of Object.entries(deltas)) {
       if (!Number.isInteger(value) || value < 0) throw new Error(`incrementCounter: invalid delta ${field}`);
       existing[field] = (existing[field] ?? 0) + value;
@@ -332,23 +428,52 @@ export function incrementCounter({ statePaths, day, event, host, provider, mode,
   });
 }
 
-export function recordFailure({ statePaths, day, event, host, provider, failureStage }) {
+export function recordFailure({ statePaths, day, pluginVersion = PLUGIN_VERSION, event, host, provider, failureStage }) {
   incrementCounter({
-    statePaths, day, event, host, provider, failureStage, deltas: { count: 1 },
+    statePaths, day, pluginVersion, event, host, provider, failureStage, deltas: { count: 1 },
   });
 }
 
 /** Queues a single non-aggregate activity marker for this UTC day and host. */
-export function recordActiveDay({ statePaths, day, pluginVersion, host }) {
+export const REASON_PREFIX = 'reason:';
+
+export function coverageRatioBucket(routed, total) {
+  if (!Number.isInteger(routed) || !Number.isInteger(total) || routed < 0 || total < routed) {
+    throw new Error('coverageRatioBucket: invalid counts');
+  }
+  if (total === 0 || routed === 0) return 'zero';
+  const ratio = routed / total;
+  if (ratio < 0.01) return 'lt_1pct';
+  if (ratio < 0.1) return '1_to_10pct';
+  if (ratio < 0.5) return '10_to_50pct';
+  if (ratio < 0.9) return '50_to_90pct';
+  return 'gt_90pct';
+}
+
+/** One call per classified shell command. Counters close into a single row per day, next to the
+ * reduction they qualify: a large saving on a small share of commands should not read the same as
+ * a large saving on all of them. */
+export function recordCoverage({ statePaths, day, pluginVersion = PLUGIN_VERSION, host, routed, reason }) {
+  const deltas = routed ? { routed: 1 } : { bypassed: 1 };
+  if (!routed) {
+    const label = COVERAGE_REASONS.includes(reason) ? reason : 'other';
+    deltas[`${REASON_PREFIX}${label}`] = 1;
+  }
+  incrementCounter({ statePaths, day, pluginVersion, event: 'coverage_summary', host, deltas });
+}
+
+export function recordActiveDay({ statePaths, day, pluginVersion = PLUGIN_VERSION, host }) {
   const marker = {
     schema_version: SCHEMA_VERSION, event: 'active_day', day_utc: day, plugin_version: pluginVersion, host,
   };
   const validatedMarker = validateEvent(marker);
-  const activeDayKey = `${day}|${host}`;
+  const activeDayKey = `${day}|${pluginVersion}|${host}`;
+  const legacyActiveDayKey = `${day}|${host}`;
   ensureDirectory(path.dirname(statePaths.counters));
   withLock(`${statePaths.counters}.lock`, () => {
     const state = readCounters(statePaths.counters);
     const activeDays = state.active_days;
+    if (Object.hasOwn(activeDays, legacyActiveDayKey)) delete activeDays[legacyActiveDayKey];
     const cutoff = Date.parse(`${day}T00:00:00Z`) - (ACTIVE_DAY_RETENTION_DAYS - 1) * 86_400_000;
     for (const [key, recordedDay] of Object.entries(activeDays)) {
       if (Date.parse(`${recordedDay}T00:00:00Z`) < cutoff) delete activeDays[key];
@@ -459,6 +584,7 @@ export function toOtlpLogs(rows) {
       resource: { attributes: [{ key: 'service.name', value: { stringValue: 'sando' } }] },
       scopeLogs: [{
         logRecords: rows.map((row) => ({
+          ...(typeof row._timeUnixNano === 'string' ? { timeUnixNano: row._timeUnixNano } : {}),
           body: { stringValue: 'sando.daily_aggregate' },
           attributes: Object.entries(publicRow(row)).map(([key, value]) => ({ key, value: { stringValue: String(value) } })),
         })),