sandoichi 0.4.1 → 0.5.0

package/src/context-capture.mjs

@@ -0,0 +1,200 @@
+import { createHash } from 'node:crypto';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+import {
+  CONTEXT_CAPTURE_SCHEMA,
+  buildContextFootprintReport,
+  serializeContextFootprint,
+} from './context-footprint.mjs';
+import { classifyContextRequest } from './context-classifier.mjs';
+
+export const CONTEXT_CAPTURE_RECORD_SCHEMA = 'sando-context-capture-record/v1';
+export const CONTEXT_CAPTURE_RECORD_VERSION = 1;
+
+const PROVIDER_FORMATS = Object.freeze({
+  anthropic: { host: 'claude', requestFormat: 'anthropic' },
+  'openai-responses': { host: 'codex', requestFormat: 'openai-responses' },
+});
+
+function object(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function counter(value) {
+  return Number.isSafeInteger(value) && value >= 0 ? value : null;
+}
+
+function add(left, right) {
+  const result = left + right;
+  return Number.isSafeInteger(result) ? result : null;
+}
+
+function optionalCounter(value) {
+  return value === undefined || value === null ? 0 : counter(value);
+}
+
+function sha256(value) {
+  return `sha256:${createHash('sha256').update(value).digest('hex')}`;
+}
+
+function isoDate(value) {
+  const date = value instanceof Date ? value : new Date(value ?? Date.now());
+  if (Number.isNaN(date.getTime())) throw new TypeError('capture timestamp is invalid');
+  return date.toISOString();
+}
+
+function safeModel(value) {
+  if (value === undefined || value === null) return null;
+  if (typeof value !== 'string' || value.length > 256) throw new TypeError('capture model is invalid');
+  return value;
+}
+
+function rawText(value) {
+  if (typeof value === 'string') return value;
+  if (Buffer.isBuffer(value) || value instanceof Uint8Array) return Buffer.from(value).toString('utf8');
+  throw new TypeError('raw request body is invalid');
+}
+
+function anthropicUsage(value) {
+  if (!object(value)) return null;
+  const input = counter(value.input_tokens);
+  const cacheRead = optionalCounter(value.cache_read_input_tokens);
+  const cacheWrite = optionalCounter(value.cache_creation_input_tokens);
+  const output = counter(value.output_tokens);
+  if (input === null || cacheRead === null || cacheWrite === null || output === null) return null;
+  const inputTokens = add(add(input, cacheRead), cacheWrite);
+  const totalTokens = inputTokens === null ? null : add(inputTokens, output);
+  if (totalTokens === null) return null;
+  const result = {
+    inputTokens,
+    cachedInputTokens: cacheRead,
+    cacheWriteInputTokens: cacheWrite,
+    cacheReadInputTokens: cacheRead,
+    outputTokens: output,
+    totalTokens,
+  };
+  if (counter(value.reasoning_output_tokens) !== null) result.reasoningOutputTokens = value.reasoning_output_tokens;
+  if (typeof value.total_cost_usd === 'number' && Number.isFinite(value.total_cost_usd) && value.total_cost_usd >= 0) {
+    result.totalCostUsd = value.total_cost_usd;
+  }
+  return result;
+}
+
+function responsesUsage(value) {
+  if (!object(value)) return null;
+  const input = counter(value.input_tokens);
+  const output = counter(value.output_tokens);
+  const cached = optionalCounter(value.cached_input_tokens
+    ?? value.cache_read_input_tokens
+    ?? value.input_tokens_details?.cached_tokens);
+  const cacheWrite = optionalCounter(value.cache_write_input_tokens);
+  const reasoning = optionalCounter(value.reasoning_output_tokens
+    ?? value.output_tokens_details?.reasoning_tokens);
+  if ([input, output, cached, cacheWrite, reasoning].some((item) => item === null)
+    || reasoning > output) return null;
+  const totalTokens = value.total_tokens === undefined ? add(input, output) : counter(value.total_tokens);
+  if (totalTokens === null || totalTokens !== input + output) return null;
+  const result = {
+    inputTokens: input,
+    cachedInputTokens: cached,
+    cacheWriteInputTokens: cacheWrite,
+    cacheReadInputTokens: cached,
+    outputTokens: output,
+    reasoningOutputTokens: reasoning,
+    totalTokens,
+  };
+  if (typeof value.total_cost_usd === 'number' && Number.isFinite(value.total_cost_usd) && value.total_cost_usd >= 0) {
+    result.totalCostUsd = value.total_cost_usd;
+  }
+  return result;
+}
+
+export function normalizeProviderUsage(provider, usage) {
+  if (usage === undefined || usage === null) return null;
+  if (provider === 'anthropic') return anthropicUsage(usage);
+  if (provider === 'openai-responses') return responsesUsage(usage);
+  return null;
+}
+
+export function defaultContextCapturePath(env = process.env) {
+  const configured = env.SANDO_CONTEXT_FOOTPRINT_PATH;
+  if (configured !== undefined) {
+    if (typeof configured !== 'string' || !path.isAbsolute(configured)) throw new Error('context capture path must be absolute');
+    return configured;
+  }
+  const stateHome = env.XDG_STATE_HOME || path.join(os.homedir(), '.local', 'state');
+  if (!path.isAbsolute(stateHome)) throw new Error('state directory must be absolute');
+  return path.join(stateHome, 'sando', 'context-footprints.jsonl');
+}
+
+export function buildContextCaptureRecord({
+  host, provider, rawBody, requestBody, sessionKey, model, providerUsage, now = new Date(), toolSearch,
+} = {}) {
+  const format = PROVIDER_FORMATS[provider];
+  if (!format || host !== format.host) throw new TypeError('provider and capture host do not match');
+  if (typeof sessionKey !== 'string' || sessionKey.length === 0) return null;
+  const content = rawText(rawBody);
+  let body = requestBody;
+  if (body === undefined) {
+    try { body = JSON.parse(content); } catch { body = null; }
+  }
+  const classification = classifyContextRequest({ provider, body });
+  const report = buildContextFootprintReport({
+    schema: CONTEXT_CAPTURE_SCHEMA,
+    host,
+    requestFormat: format.requestFormat,
+    body: { state: 'observed', content },
+    segments: classification.segments,
+    providerUsage: normalizeProviderUsage(provider, providerUsage),
+    toolSearch: toolSearch ?? { state: 'indeterminate' },
+  });
+  return {
+    schema: CONTEXT_CAPTURE_RECORD_SCHEMA,
+    version: CONTEXT_CAPTURE_RECORD_VERSION,
+    at: isoDate(now),
+    host,
+    provider,
+    requestFormat: format.requestFormat,
+    model: safeModel(model),
+    sessionKeyDigest: sha256(sessionKey),
+    report,
+  };
+}
+
+function ensureDirectory(directory) {
+  fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
+  const stat = fs.lstatSync(directory);
+  if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error('context capture directory is unsafe');
+  fs.chmodSync(directory, 0o700);
+}
+
+function validateRecord(record) {
+  if (!object(record)
+    || record.schema !== CONTEXT_CAPTURE_RECORD_SCHEMA
+    || record.version !== CONTEXT_CAPTURE_RECORD_VERSION
+    || typeof record.at !== 'string'
+    || typeof record.host !== 'string'
+    || typeof record.provider !== 'string'
+    || typeof record.requestFormat !== 'string'
+    || !/^sha256:[0-9a-f]{64}$/.test(record.sessionKeyDigest)
+    || !object(record.report)
+    || record.report.schema !== 'sando-context-footprint/v1') {
+    throw new TypeError('context capture record is invalid');
+  }
+  serializeContextFootprint(record.report);
+}
+
+export function recordContextCapture({ storagePath, record } = {}) {
+  if (typeof storagePath !== 'string' || !path.isAbsolute(storagePath)) throw new Error('context capture path must be absolute');
+  validateRecord(record);
+  ensureDirectory(path.dirname(storagePath));
+  if (fs.existsSync(storagePath)) {
+    const stat = fs.lstatSync(storagePath);
+    if (!stat.isFile() || stat.isSymbolicLink()) throw new Error('context capture file is unsafe');
+  }
+  fs.appendFileSync(storagePath, `${JSON.stringify(record)}\n`, { mode: 0o600 });
+  fs.chmodSync(storagePath, 0o600);
+  return record;
+}

package/src/context-classifier.mjs

@@ -0,0 +1,142 @@
+import { CONTEXT_CATEGORIES } from './context-footprint.mjs';
+
+const CATEGORY_SET = new Set(CONTEXT_CATEGORIES);
+
+function object(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function jsonBytes(value) {
+  try {
+    const text = JSON.stringify(value);
+    return text === undefined ? 0 : Buffer.byteLength(text, 'utf8');
+  } catch {
+    return 0;
+  }
+}
+
+function nameOf(value) {
+  return typeof value?.name === 'string' ? value.name : '';
+}
+
+function deferred(value) {
+  return value?.defer_loading === true
+    || value?.deferred === true
+    || value?.deferLoading === true;
+}
+
+function toolCategory(tool) {
+  const name = nameOf(tool).toLowerCase();
+  if (name.includes('sando') || name.startsWith('sando_')) return 'sando';
+  if (deferred(tool)) return 'mcp-deferred';
+  if (name.startsWith('mcp__') || name.startsWith('mcp_')) return 'mcp-direct';
+  return 'builtin-tools';
+}
+
+function without(value, keys) {
+  if (!object(value)) return value;
+  const result = { ...value };
+  for (const key of keys) delete result[key];
+  return result;
+}
+
+function hasToolOutput(item) {
+  const type = typeof item?.type === 'string' ? item.type.toLowerCase() : '';
+  return item?.role === 'tool'
+    || type === 'tool_result'
+    || type.endsWith('_tool_call_output')
+    || type.endsWith('function_call_output')
+    || Object.hasOwn(item ?? {}, 'output');
+}
+
+function addSegment(segments, budget, category, value) {
+  if (!CATEGORY_SET.has(category)) return budget;
+  const bytes = Math.min(jsonBytes(value), budget);
+  if (bytes > 0) segments.push({ category, bytes });
+  return budget - bytes;
+}
+
+function classifyAnthropic(body, segments, budget) {
+  const handled = new Set(['system', 'tools', 'messages']);
+  if (Object.hasOwn(body, 'system')) budget = addSegment(segments, budget, 'host-instructions', body.system);
+
+  if (Array.isArray(body.tools)) {
+    for (const tool of body.tools) {
+      budget = addSegment(segments, budget, toolCategory(tool), tool);
+    }
+  }
+
+  if (Array.isArray(body.messages)) {
+    let currentUser = -1;
+    body.messages.forEach((message, index) => {
+      if (message?.role === 'user' && !hasToolOutput(message)) currentUser = index;
+    });
+    body.messages.forEach((message, index) => {
+      const category = index === currentUser ? 'user-prompt' : 'history';
+      budget = addSegment(segments, budget, category, message);
+    });
+  }
+
+  for (const [key, value] of Object.entries(body)) {
+    if (!handled.has(key)) budget = addSegment(segments, budget, 'provider-overhead', value);
+  }
+  return budget;
+}
+
+function codexMessageCategory(item, currentUser) {
+  if (item?.role === 'developer' || item?.role === 'system') return 'host-instructions';
+  if (currentUser && item === currentUser) return 'user-prompt';
+  return 'history';
+}
+
+function classifyCodex(body, segments, budget) {
+  const handled = new Set(['instructions', 'input', 'tools']);
+  if (Object.hasOwn(body, 'instructions')) budget = addSegment(segments, budget, 'host-instructions', body.instructions);
+
+  if (Array.isArray(body.tools)) {
+    for (const tool of body.tools) budget = addSegment(segments, budget, toolCategory(tool), tool);
+  }
+
+  if (Array.isArray(body.input)) {
+    let currentUser = null;
+    for (const item of body.input) {
+      if (object(item) && item.role === 'user' && !hasToolOutput(item)) currentUser = item;
+    }
+    for (const item of body.input) {
+      if (!object(item)) {
+        budget = addSegment(segments, budget, 'provider-overhead', item);
+        continue;
+      }
+      if (Array.isArray(item.tools)) {
+        for (const tool of item.tools) budget = addSegment(segments, budget, toolCategory(tool), tool);
+      }
+      const message = Object.hasOwn(item, 'content')
+        ? without(item, ['content', 'tools'])
+        : without(item, ['tools']);
+      const messageCategory = hasToolOutput(item)
+        ? 'history'
+        : codexMessageCategory(item, currentUser);
+      if (Object.hasOwn(item, 'content')) {
+        budget = addSegment(segments, budget, messageCategory, item.content);
+        budget = addSegment(segments, budget, 'provider-overhead', message);
+      } else {
+        budget = addSegment(segments, budget, 'provider-overhead', message);
+      }
+    }
+  }
+
+  for (const [key, value] of Object.entries(body)) {
+    if (!handled.has(key)) budget = addSegment(segments, budget, 'provider-overhead', value);
+  }
+  return budget;
+}
+
+export function classifyContextRequest({ provider, body } = {}) {
+  const segments = [];
+  if (!object(body)) return { segments };
+  const bodyBytes = jsonBytes(body);
+  let budget = bodyBytes;
+  if (provider === 'anthropic') budget = classifyAnthropic(body, segments, budget);
+  else if (provider === 'openai-responses') budget = classifyCodex(body, segments, budget);
+  return { segments, unclassifiedBytes: budget };
+}

package/src/context-footprint.mjs

@@ -0,0 +1,299 @@
+import { createHash } from 'node:crypto';
+
+import { createRedactionProfile } from './redaction-profile.mjs';
+
+export const CONTEXT_CAPTURE_SCHEMA = 'sando-context-capture/v1';
+export const CONTEXT_FOOTPRINT_SCHEMA = 'sando-context-footprint/v1';
+export const CONTEXT_FOOTPRINT_VERSION = 1;
+
+export const CONTEXT_CATEGORIES = Object.freeze([
+  'host-instructions',
+  'project-instructions',
+  'skills',
+  'builtin-tools',
+  'mcp-direct',
+  'mcp-deferred',
+  'sando',
+  'user-prompt',
+  'history',
+  'provider-overhead',
+  'unknown',
+]);
+
+const HOSTS = new Set(['claude', 'codex']);
+const BODY_STATES = new Set(['observed', 'partial', 'unavailable']);
+const TOOL_SEARCH_STATES = new Set(['enabled', 'disabled', 'unavailable', 'indeterminate']);
+const FORMATS = Object.freeze({
+  claude: new Set(['anthropic']),
+  codex: new Set(['openai-responses', 'codex-cli']),
+});
+const PROVIDER_FIELDS = Object.freeze([
+  'inputTokens',
+  'cachedInputTokens',
+  'cacheWriteInputTokens',
+  'cacheReadInputTokens',
+  'outputTokens',
+  'reasoningOutputTokens',
+  'totalTokens',
+  'totalCostUsd',
+]);
+const DEFAULT_PROFILE = createRedactionProfile();
+
+function object(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function counter(value, name) {
+  if (!Number.isSafeInteger(value) || value < 0) throw new TypeError(`${name} must be a non-negative safe integer`);
+  return value;
+}
+
+function add(left, right, name) {
+  const total = left + right;
+  if (!Number.isSafeInteger(total)) throw new RangeError(`${name} exceeds safe integer range`);
+  return total;
+}
+
+function digest(value) {
+  if (typeof value !== 'string' || !/^sha256:[0-9a-f]{64}$/.test(value)) throw new TypeError('digest is invalid');
+  return value;
+}
+
+function sha256(value) {
+  return `sha256:${createHash('sha256').update(value).digest('hex')}`;
+}
+
+function stableJson(value, seen = new Set()) {
+  if (value === null || typeof value !== 'object') {
+    const result = JSON.stringify(value);
+    if (result === undefined) throw new TypeError('value is not JSON serializable');
+    return result;
+  }
+  if (seen.has(value)) throw new TypeError('value must not be cyclic');
+  seen.add(value);
+  const result = Array.isArray(value)
+    ? `[${value.map((item) => stableJson(item, seen)).join(',')}]`
+    : `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson(value[key], seen)}`).join(',')}}`;
+  seen.delete(value);
+  return result;
+}
+
+function estimateBytes(bytes) {
+  return bytes === 0 ? 0 : Math.ceil(bytes / 4);
+}
+
+function profileFor(candidate) {
+  if (candidate === undefined) return DEFAULT_PROFILE;
+  if (!object(candidate) || typeof candidate.redact !== 'function') throw new TypeError('redactionProfile is invalid');
+  return candidate;
+}
+
+function redactedDigest(text, profile) {
+  return sha256(profile.redact(text).text);
+}
+
+function contentEvidence(value, name, profile) {
+  if (typeof value !== 'string') throw new TypeError(`${name}.content must be a string`);
+  return { bytes: Buffer.byteLength(value, 'utf8'), digest: redactedDigest(value, profile) };
+}
+
+function bodyEvidence(body, profile) {
+  if (!object(body)) throw new TypeError('body is invalid');
+  const state = body.state ?? 'unavailable';
+  if (!BODY_STATES.has(state)) throw new TypeError('body state is invalid');
+  if (state === 'unavailable') return { state, bytes: null, digest: null };
+
+  const content = Object.hasOwn(body, 'content') ? contentEvidence(body.content, 'body', profile) : null;
+  const bytes = content ? content.bytes : counter(body.bytes, 'body.bytes');
+  if (Object.hasOwn(body, 'bytes') && counter(body.bytes, 'body.bytes') !== bytes) {
+    throw new TypeError('body bytes contradict content');
+  }
+  const bodyDigest = content ? content.digest : (body.digest === undefined ? null : digest(body.digest));
+  if (content && body.digest !== undefined && digest(body.digest) !== bodyDigest) {
+    throw new TypeError('body digest contradicts content');
+  }
+  return { state, bytes, digest: bodyDigest };
+}
+
+function categoryOf(value) {
+  if (typeof value !== 'string' || !CONTEXT_CATEGORIES.includes(value)) throw new TypeError('segment category is invalid');
+  return value;
+}
+
+function normalizeSegments(segments, profile) {
+  if (segments === undefined) return [];
+  if (!Array.isArray(segments) || segments.length > 10_000) throw new TypeError('segments are invalid');
+  return segments.map((segment) => {
+    if (!object(segment)) throw new TypeError('segment is invalid');
+    const category = categoryOf(segment.category);
+    const content = Object.hasOwn(segment, 'content') ? contentEvidence(segment.content, 'segment', profile) : null;
+    const bytes = content ? content.bytes : counter(segment.bytes, 'segment.bytes');
+    if (Object.hasOwn(segment, 'bytes') && counter(segment.bytes, 'segment.bytes') !== bytes) {
+      throw new TypeError('segment bytes contradict content');
+    }
+    return { category, bytes, digest: content?.digest ?? null };
+  });
+}
+
+function validateSegmentMembership(body, segments) {
+  if (typeof body?.content !== 'string' || !Array.isArray(segments)) return;
+  let cursor = 0;
+  for (const segment of segments) {
+    if (!object(segment) || typeof segment.content !== 'string') continue;
+    const position = body.content.indexOf(segment.content, cursor);
+    if (position < 0) throw new TypeError('segment content is not a non-overlapping part of the observed body');
+    cursor = position + segment.content.length;
+  }
+}
+
+function providerUsage(value) {
+  if (value === undefined || value === null) return null;
+  if (!object(value)) throw new TypeError('providerUsage is invalid');
+  const result = { source: 'provider-reported' };
+  for (const field of PROVIDER_FIELDS) {
+    if (!Object.hasOwn(value, field)) continue;
+    if (field === 'totalCostUsd') {
+      if (typeof value[field] !== 'number' || !Number.isFinite(value[field]) || value[field] < 0) {
+        throw new TypeError('provider cost is invalid');
+      }
+      result[field] = value[field];
+    } else result[field] = counter(value[field], `providerUsage.${field}`);
+  }
+  if (!Object.hasOwn(result, 'inputTokens')) throw new TypeError('providerUsage.inputTokens is required');
+  if (Object.hasOwn(result, 'cachedInputTokens') && Object.hasOwn(result, 'cacheReadInputTokens')
+    && result.cachedInputTokens !== result.cacheReadInputTokens) {
+    throw new TypeError('provider cache-read counters contradict each other');
+  }
+  const cacheRead = result.cacheReadInputTokens ?? result.cachedInputTokens;
+  if ([cacheRead, result.cacheWriteInputTokens].some((value) => value !== undefined && value > result.inputTokens)
+    || (cacheRead !== undefined && result.cacheWriteInputTokens !== undefined
+      && add(cacheRead, result.cacheWriteInputTokens, 'provider cache counters') > result.inputTokens)) {
+    throw new TypeError('provider cache counters exceed input tokens');
+  }
+  if (Object.hasOwn(result, 'reasoningOutputTokens') && Object.hasOwn(result, 'outputTokens')
+    && result.reasoningOutputTokens > result.outputTokens) {
+    throw new TypeError('provider reasoning tokens exceed output tokens');
+  }
+  if (Object.hasOwn(result, 'outputTokens') && Object.hasOwn(result, 'totalTokens')
+    && add(result.inputTokens, result.outputTokens, 'provider token counters') !== result.totalTokens) {
+    throw new TypeError('provider total tokens are invalid');
+  }
+  return result;
+}
+
+export function detectToolSearchState(value) {
+  if (!object(value)) return 'indeterminate';
+  if (TOOL_SEARCH_STATES.has(value.state)) return value.state;
+  if (typeof value.enabled === 'boolean') return value.enabled ? 'enabled' : 'disabled';
+  if (value.available === false) return 'unavailable';
+  return 'indeterminate';
+}
+
+function emptyCategories() {
+  return Object.fromEntries(CONTEXT_CATEGORIES.map((category) => [category, {
+    bytes: 0, estimatedTokens: 0, segmentCount: 0,
+  }]));
+}
+
+function categorySummary(segments) {
+  const categories = emptyCategories();
+  for (const segment of segments) {
+    const current = categories[segment.category];
+    current.bytes = add(current.bytes, segment.bytes, 'category bytes');
+    current.estimatedTokens = estimateBytes(current.bytes);
+    current.segmentCount += 1;
+  }
+  return categories;
+}
+
+function safeReport(report) {
+  const result = { ...report };
+  delete result.provenanceDigest;
+  return result;
+}
+
+export function serializeContextFootprint(report) {
+  if (!object(report) || report.schema !== CONTEXT_FOOTPRINT_SCHEMA) throw new TypeError('context footprint report is invalid');
+  return stableJson(report);
+}
+
+export function buildContextFootprintReport(capture, { redactionProfile } = {}) {
+  if (!object(capture) || capture.schema !== CONTEXT_CAPTURE_SCHEMA) throw new TypeError('context capture schema is invalid');
+  if (!HOSTS.has(capture.host)) throw new TypeError('context capture host is invalid');
+  const profile = profileFor(redactionProfile);
+  validateSegmentMembership(capture.body, capture.segments);
+  const body = bodyEvidence(capture.body ?? { state: 'unavailable' }, profile);
+  const requestFormat = capture.requestFormat ?? null;
+  if (requestFormat !== null && (typeof requestFormat !== 'string' || !FORMATS[capture.host].has(requestFormat))) {
+    throw new TypeError('context capture request format is invalid');
+  }
+  if (requestFormat === null && body.state !== 'unavailable') {
+    throw new TypeError('context capture request format is required for an observed body');
+  }
+  const segments = normalizeSegments(capture.segments, profile);
+  const providerReported = providerUsage(capture.providerUsage);
+  const toolSearch = { state: detectToolSearchState(capture.toolSearch) };
+  const base = {
+    schema: CONTEXT_FOOTPRINT_SCHEMA,
+    version: CONTEXT_FOOTPRINT_VERSION,
+    host: capture.host,
+    requestFormat,
+    toolSearch,
+    observation: { status: body.state, bodyDigest: body.digest },
+    attribution: null,
+    categories: null,
+    tokenAccounting: {
+      estimated: {
+        source: 'mechanical-estimate',
+        formula: 'ceil(UTF-8 bytes / 4)',
+        totalTokens: null,
+        attributedTokens: null,
+        unknownTokens: null,
+        categories: null,
+      },
+      providerReported,
+    },
+  };
+
+  if (body.state !== 'unavailable') {
+    const categories = categorySummary(segments);
+    const observedBytes = segments.reduce((total, segment) => add(total, segment.bytes, 'observed bytes'), 0);
+    if (observedBytes > body.bytes) throw new RangeError('segments exceed body bytes');
+    const unclassifiedBytes = body.bytes - observedBytes;
+    categories.unknown.bytes = add(categories.unknown.bytes, unclassifiedBytes, 'unknown bytes');
+    categories.unknown.estimatedTokens = estimateBytes(categories.unknown.bytes);
+    const unknownBytes = categories.unknown.bytes;
+    const attributedBytes = body.bytes - unknownBytes;
+    const categoryTokens = Object.fromEntries(
+      CONTEXT_CATEGORIES.map((category) => [category, categories[category].estimatedTokens]),
+    );
+    base.attribution = {
+      status: body.state === 'partial' || unknownBytes > 0 ? 'partial' : 'complete',
+      bodyBytes: body.bytes,
+      observedBytes,
+      attributedBytes,
+      unknownBytes,
+      unknownRatio: body.bytes === 0 ? 0 : unknownBytes / body.bytes,
+    };
+    base.categories = categories;
+    base.tokenAccounting.estimated = {
+      ...base.tokenAccounting.estimated,
+      totalTokens: estimateBytes(body.bytes),
+      attributedTokens: estimateBytes(attributedBytes),
+      unknownTokens: estimateBytes(unknownBytes),
+      categories: categoryTokens,
+    };
+  } else {
+    base.attribution = {
+      status: 'unavailable',
+      bodyBytes: null,
+      observedBytes: null,
+      attributedBytes: null,
+      unknownBytes: null,
+      unknownRatio: null,
+    };
+  }
+
+  base.provenanceDigest = sha256(serializeContextFootprint(safeReport(base)));
+  return base;
+}