tledger 0.3.1 → 0.4.0

package/bin/token-ledger-source-status.mjs

@@ -0,0 +1,31 @@
+export const SOURCE_STATUSES = Object.freeze([
+  "verified-current",
+  "explicit-snapshot",
+  "unchecked-cache",
+  "stale-fallback",
+]);
+
+const SOURCE_STATUS_LABELS = Object.freeze({
+  "verified-current": "VERIFIED CURRENT",
+  "stale-fallback": "STALE FALLBACK",
+  "unchecked-cache": "UNCHECKED CACHE",
+  "explicit-snapshot": "EXPLICIT SNAPSHOT",
+});
+
+export function sourceStatusLabel(sourceStatus = "unchecked-cache") {
+  const label = SOURCE_STATUS_LABELS[sourceStatus];
+  if (!label) {
+    throw new Error(`Unknown report source status: ${String(sourceStatus)}`);
+  }
+  return label;
+}
+
+export function sourceStatusLine(sourceStatus = "unchecked-cache") {
+  return `PROVENANCE · ${sourceStatusLabel(sourceStatus)}`;
+}
+
+export function snapshotFreshnessDetail(freshness) {
+  return freshness?.status === "fresh" || freshness?.status === "stale"
+    ? `${freshness.status} · ${freshness.ageLabel}`
+    : "age unknown";
+}

package/bin/token-ledger-terminal.mjs

@@ -8,10 +8,22 @@ import {
   INTERACTIVE_HELP,
 } from "./token-ledger-controls.mjs";
 import {
+  MAX_SAFE_TOKEN_COUNT,
+  checkedFiniteAdd,
+  checkedTokenAdd,
+  scaledOutputTokens,
+  tokenValue,
   usageBucketsInRange,
   usageCallCount,
   usageThreadIds,
 } from "../lib/token-ledger-usage.mjs";
+import { calendarDateParts } from "../lib/token-ledger-calendar.mjs";
+import { sanitizeTerminalText } from "../lib/token-ledger-terminal-text.mjs";
+import {
+  snapshotFreshnessDetail,
+  sourceStatusLine,
+} from "./token-ledger-source-status.mjs";
+import { historyScopeLabel } from "../lib/token-ledger-collection.mjs";
 
 const RESET = "\u001b[0m";
 const PRIMARY_STYLE = [38, 2, 255, 255, 255];
@@ -20,6 +32,7 @@ const ACCENT_STYLE = [38, 2, 51, 156, 255];
 const BORDER_STYLE = [38, 2, 88, 88, 88];
 const TRACK_STYLE = [38, 2, 59, 59, 59];
 export const MODEL_COLORS = {
+  astra: [38, 2, 232, 121, 249],
   sol: [38, 2, 120, 185, 242],
   luna: ACCENT_STYLE,
   terra: [38, 2, 214, 168, 95],
@@ -41,7 +54,7 @@ function colorize(value, code, enabled) {
 }
 
 function stripAnsi(value) {
-  return String(value).replace(/\u001b\[[0-9;]*m/g, "");
+  return sanitizeTerminalText(value);
 }
 
 function visibleLength(value) {
@@ -111,6 +124,7 @@ function plural(value, singular, pluralForm = `${singular}s`) {
 function modelLabel(value) {
   const model = String(value || "Unknown model");
   const lower = model.toLowerCase();
+  if (lower.includes("astra")) return "Astra";
   if (lower.includes("sol")) return "Sol";
   if (lower.includes("luna")) return "Luna";
   if (lower.includes("terra")) return "Terra";
@@ -124,7 +138,7 @@ function modelColor(model) {
 }
 
 function usageTypeLabel(value) {
-  const words = String(value || "unknown")
+  const words = sanitizeTerminalText(value || "unknown")
     .trim()
     .replace(/[_-]+/g, " ")
     .split(/\s+/)
@@ -140,7 +154,12 @@ function usageTypeLabel(value) {
 }
 
 function displayProject(row) {
-  return row.displayProject || row.project || "Unlabelled activity";
+  const label = sanitizeTerminalText(
+    row.displayProject || row.project || "Unlabelled activity",
+  )
+    .replace(/\s+/g, " ")
+    .trim();
+  return label || "Unlabelled activity";
 }
 
 function isRollingRange(range) {
@@ -161,26 +180,37 @@ function dateLabel(bounds, range = "day", rollingLabel = "1 day") {
     const end = `${monthNames[endParts[1] - 1]} ${String(endParts[2]).padStart(2, "0")}`;
     return `${start} – ${end} ${endParts[0]}`;
   }
-  const parts = new Intl.DateTimeFormat("en-US", {
-    timeZone: bounds.timeZone,
+  const values = calendarDateParts(bounds.dateString, {
     weekday: "short",
     day: "2-digit",
     month: "short",
     year: "numeric",
-  }).formatToParts(bounds.start);
-  const values = Object.fromEntries(
-    parts
-      .filter((part) => part.type !== "literal")
-      .map((part) => [part.type, part.value]),
-  );
+  });
   return `${values.weekday} ${values.day} ${values.month} ${values.year}`.toUpperCase();
 }
 
 function modelTotals(events) {
   const totals = new Map();
+  let scale = 1;
+  let totalTokens = 0;
   for (const event of events) {
+    if (event?.invalidTokenRecord === true) continue;
     const model = modelLabel(event.model);
-    totals.set(model, (totals.get(model) ?? 0) + (Number(event.totalTokens) || 0));
+    const allowFractional = event.rangeAllocationEstimated === true;
+    const tokens = tokenValue(event.totalTokens, { allowFractional });
+    const scaledTokens = tokens / scale;
+    totalTokens += scaledTokens;
+    totals.set(
+      model,
+      (totals.get(model) ?? 0) + scaledTokens,
+    );
+    const scaleFactor = Math.max(1, totalTokens / MAX_SAFE_TOKEN_COUNT);
+    if (scaleFactor === 1) continue;
+    for (const [modelName, value] of totals) {
+      totals.set(modelName, value / scaleFactor);
+    }
+    totalTokens = MAX_SAFE_TOKEN_COUNT;
+    scale *= scaleFactor;
   }
   return [...totals.entries()]
     .map(([model, totalTokens]) => ({ model, totalTokens }))
@@ -189,9 +219,28 @@ function modelTotals(events) {
 
 function usageTypeTotals(events) {
   const totals = new Map();
+  let scale = 1;
+  let totalTokens = 0;
   for (const event of events) {
-    const key = String(event.useType || "unknown").trim().toLowerCase() || "unknown";
-    totals.set(key, (totals.get(key) ?? 0) + (Number(event.totalTokens) || 0));
+    if (event?.invalidTokenRecord === true) continue;
+    const key = sanitizeTerminalText(event.useType || "unknown")
+      .trim()
+      .toLowerCase() || "unknown";
+    const allowFractional = event.rangeAllocationEstimated === true;
+    const tokens = tokenValue(event.totalTokens, { allowFractional });
+    const scaledTokens = tokens / scale;
+    totalTokens += scaledTokens;
+    totals.set(
+      key,
+      (totals.get(key) ?? 0) + scaledTokens,
+    );
+    const scaleFactor = Math.max(1, totalTokens / MAX_SAFE_TOKEN_COUNT);
+    if (scaleFactor === 1) continue;
+    for (const [usageType, value] of totals) {
+      totals.set(usageType, value / scaleFactor);
+    }
+    totalTokens = MAX_SAFE_TOKEN_COUNT;
+    scale *= scaleFactor;
   }
   return [...totals.entries()]
     .map(([key, totalTokens]) => ({
@@ -202,6 +251,19 @@ function usageTypeTotals(events) {
     .sort((left, right) => right.totalTokens - left.totalTokens);
 }
 
+function addCacheTotals(state, inputTokens, cachedInputTokens) {
+  const nextInput = state.inputTokens + inputTokens / state.scale;
+  const nextCached = state.cachedInputTokens + cachedInputTokens / state.scale;
+  const scaleFactor = Math.max(
+    1,
+    nextInput / MAX_SAFE_TOKEN_COUNT,
+    nextCached / MAX_SAFE_TOKEN_COUNT,
+  );
+  state.inputTokens = nextInput / scaleFactor;
+  state.cachedInputTokens = nextCached / scaleFactor;
+  state.scale *= scaleFactor;
+}
+
 function latestWeeklyQuotaObservation(snapshot) {
   // The epoch-keyed timeline drops stale readings from superseded windows,
   // so the last entry is the newest reading of the currently-live window.
@@ -228,9 +290,12 @@ export function quotaCycleSummary(snapshot = {}, displayedEvents = []) {
   const windowStartMs =
     (Number(observation.resetsAt) - Number(observation.windowMinutes) * 60) * 1_000;
   const resetAtMs = Number(observation.resetsAt) * 1_000;
+  const observationThroughMs = Number.isFinite(observation.observedThroughMs)
+    ? observation.observedThroughMs
+    : new Date(observation.eventCutoffAt ?? observation.timestamp).getTime();
   const observedThroughMs = Math.min(
     resetAtMs,
-    new Date(observation.eventCutoffAt ?? observation.timestamp).getTime(),
+    observationThroughMs,
   );
   if (
     !Number.isFinite(windowStartMs) ||
@@ -249,20 +314,43 @@ export function quotaCycleSummary(snapshot = {}, displayedEvents = []) {
     };
   }
 
-  const sumUsage = (events) =>
-    events.reduce(
-      (acc, event) => {
-        const tokens = Number(event.totalTokens) || 0;
-        acc.tokens += tokens;
-        const credits = eventCredits(event);
-        if (Number.isFinite(credits) && credits >= 0) {
-          acc.credits += credits;
-          acc.ratedTokens += tokens;
-        }
-        return acc;
-      },
-      { tokens: 0, credits: 0, ratedTokens: 0 },
-    );
+  const sumUsage = (events, initialScale = 1) => {
+    const acc = {
+      tokens: 0,
+      credits: 0,
+      ratedTokens: 0,
+      hasUnrated: false,
+      scale: Number.isFinite(initialScale) && initialScale >= 1
+        ? initialScale
+        : 1,
+    };
+    for (const event of events) {
+      if (event?.invalidTokenRecord === true) continue;
+      const allowFractional = event.rangeAllocationEstimated === true;
+      const tokens = tokenValue(event.totalTokens, { allowFractional });
+      const contribution = tokens / acc.scale;
+      const nextTokens = acc.tokens + contribution;
+      const scaleFactor = Math.max(
+        1,
+        nextTokens / MAX_SAFE_TOKEN_COUNT,
+      );
+      if (scaleFactor > 1) {
+        acc.tokens /= scaleFactor;
+        acc.ratedTokens /= scaleFactor;
+        acc.scale *= scaleFactor;
+      }
+      const scaledContribution = contribution / scaleFactor;
+      acc.tokens += scaledContribution;
+      const credits = eventCredits(event);
+      if (Number.isFinite(credits) && credits >= 0) {
+        acc.credits = checkedFiniteAdd(acc.credits, credits);
+        acc.ratedTokens += scaledContribution;
+      } else if (tokens > 0) {
+        acc.hasUnrated = true;
+      }
+    }
+    return acc;
+  };
   const cycleEndMs = observedThroughMs + 1;
   const cycle = sumUsage(
     usageBucketsInRange(snapshot, windowStartMs, cycleEndMs),
@@ -273,6 +361,7 @@ export function quotaCycleSummary(snapshot = {}, displayedEvents = []) {
       windowStartMs,
       cycleEndMs,
     ),
+    cycle.scale,
   );
   const usedPercent = Math.min(100, Math.max(0, Number(observation.usedPercent) || 0));
   // The weekly meter weights usage by model, token type, and fast mode;
@@ -280,7 +369,10 @@ export function quotaCycleSummary(snapshot = {}, displayedEvents = []) {
   // when every event in the cycle is rated, and fall back to raw token
   // share otherwise.
   const creditsUsable =
-    cycle.tokens > 0 && cycle.ratedTokens === cycle.tokens && cycle.credits > 0;
+    !cycle.hasUnrated &&
+    cycle.tokens > 0 &&
+    cycle.ratedTokens === cycle.tokens &&
+    cycle.credits > 0;
   const displayedSharePercent = creditsUsable
     ? (displayed.credits / cycle.credits) * 100
     : cycle.tokens
@@ -302,39 +394,44 @@ export function quotaCycleSummary(snapshot = {}, displayedEvents = []) {
   };
 }
 
-function summary(events) {
-  const totalTokens = events.reduce(
-    (sum, event) => sum + (Number(event.totalTokens) || 0),
+function summary(events, projectRows) {
+  const totalTokens = projectRows.reduce(
+    (sum, row) => sum + row.totalTokens,
     0,
   );
   const calls = events.reduce(
-    (sum, event) => sum + usageCallCount(event),
+    (sum, event) => {
+      const allowFractional = event?.rangeAllocationEstimated === true;
+      return checkedTokenAdd(sum, usageCallCount(event), { allowFractional });
+    },
     0,
   );
   const threadIds = new Set(
     events.flatMap((event) => usageThreadIds(event)),
   );
-  const outputTokens = events.reduce(
-    (sum, event) => sum + (Number(event.outputTokens) || 0),
-    0,
-  );
-  const inputTokens = events.reduce(
-    (sum, event) => sum + (Number(event.inputTokens) || 0),
-    0,
-  );
-  const cachedInputTokens = events.reduce((sum, event) => {
-    const input = Math.max(0, Number(event.inputTokens) || 0);
-    const cached = Math.max(0, Number(event.cachedInputTokens) || 0);
-    return sum + Math.min(input, cached);
-  }, 0);
+  const outputTokens = scaledOutputTokens(events, totalTokens);
+  const cacheTotals = { inputTokens: 0, cachedInputTokens: 0, scale: 1 };
+  for (const event of events) {
+    if (event?.invalidTokenRecord === true) continue;
+    const allowFractional = event.rangeAllocationEstimated === true;
+    const input = tokenValue(event.inputTokens, { allowFractional });
+    const cached = Math.min(
+      input,
+      tokenValue(event.cachedInputTokens, { allowFractional }),
+    );
+    addCacheTotals(cacheTotals, input, cached);
+  }
   return {
     totalTokens,
     calls,
     threads: threadIds.size,
     outputTokens,
-    inputTokens,
-    cachedInputTokens,
-    uncachedInputTokens: Math.max(0, inputTokens - cachedInputTokens),
+    inputTokens: cacheTotals.inputTokens,
+    cachedInputTokens: cacheTotals.cachedInputTokens,
+    uncachedInputTokens: Math.max(
+      0,
+      cacheTotals.inputTokens - cacheTotals.cachedInputTokens,
+    ),
     models: modelTotals(events),
     usageTypes: usageTypeTotals(events),
   };
@@ -343,12 +440,12 @@ function summary(events) {
 function modelLegendItems(models, totalTokens) {
   const known = new Map();
   for (const model of models) {
-    const key = ["Sol", "Luna", "Terra", "GPT"].includes(model.model)
+    const key = ["Astra", "Sol", "Luna", "Terra", "GPT"].includes(model.model)
       ? model.model
       : "Other";
     known.set(key, (known.get(key) ?? 0) + model.totalTokens);
   }
-  return ["Luna", "Sol", "Terra", "GPT", "Other"]
+  return ["Astra", "Luna", "Sol", "Terra", "GPT", "Other"]
     .map((model) => ({
       model,
       totalTokens: known.get(model) ?? 0,
@@ -407,7 +504,10 @@ function panelLines(rows, allRows, totalTokens, panelWidth, options, enabled) {
     panelWidth - labelWidth - shareWidth - totalWidth - 1 - rightPadding,
   );
   const maxTokens = allRows[0]?.totalTokens ?? 0;
-  const totalCredits = allRows.reduce((sum, item) => sum + item.rateCardCredits, 0) || 1;
+  const totalCredits = allRows.reduce(
+    (sum, item) => checkedFiniteAdd(sum, item.rateCardCredits),
+    0,
+  ) || 1;
   const selectedIndex = Math.min(
     Math.max(0, Math.trunc(Number(options.selectedIndex) || 0)),
     Math.max(0, rows.length - 1),
@@ -486,7 +586,12 @@ function sidebarLines(stats, panelWidth, enabled, options = {}, quota = null) {
           label: "Other",
           totalTokens: stats.usageTypes
             .slice(4)
-            .reduce((sum, item) => sum + item.totalTokens, 0),
+            .reduce(
+              (sum, item) => checkedTokenAdd(sum, item.totalTokens, {
+                allowFractional: true,
+              }),
+              0,
+            ),
         },
       ]
     : stats.usageTypes;
@@ -546,13 +651,61 @@ function panel(leftLines, rightLines, leftWidth, rightWidth, enabled, ascii) {
 }
 
 function snapshotLine(freshness, enabled) {
-  const detail = freshness?.status === "fresh" || freshness?.status === "stale"
-    ? `${freshness.status} · ${freshness.ageLabel}`
-    : "age unknown";
+  const detail = snapshotFreshnessDetail(freshness);
   return `${colorize("SNAPSHOT", ACCENT_STYLE, enabled)} ${colorize("·", SECONDARY_STYLE, enabled)} ${colorize(detail, SECONDARY_STYLE, enabled)}`;
 }
 
-function headerLines(stats, bounds, frameWidth, options, enabled, freshness) {
+function provenanceLine(sourceStatus, enabled) {
+  return colorize(sourceStatusLine(sourceStatus), SECONDARY_STYLE, enabled);
+}
+
+function positiveCoverageCount(value) {
+  try {
+    const count = Number(value);
+    return Number.isFinite(count) && count > 0 ? count : 0;
+  } catch {
+    return 0;
+  }
+}
+
+export function incompleteSourceWarning(snapshot = {}) {
+  const coverage = snapshot?.coverage ?? {};
+  const parseErrors = positiveCoverageCount(coverage.parseErrors);
+  const invalidTokenRecords = positiveCoverageCount(coverage.invalidTokenRecords);
+  if (
+    parseErrors <= 0 &&
+    invalidTokenRecords <= 0 &&
+    coverage.sourceIncomplete !== true
+  ) {
+    return null;
+  }
+  const details = [];
+  if (parseErrors > 0) {
+    details.push(
+      `${parseErrors.toLocaleString("en-US")} PARSE ERROR${parseErrors === 1 ? "" : "S"}`,
+    );
+  }
+  if (invalidTokenRecords > 0) {
+    details.push(
+      `${invalidTokenRecords.toLocaleString("en-US")} INVALID TOKEN RECORD${invalidTokenRecords === 1 ? "" : "S"}`,
+    );
+  }
+  if (coverage.sourceIncomplete === true) {
+    details.push("INCOMPLETE SOURCE PROVENANCE");
+  }
+  return ["SOURCES INCOMPLETE", ...details].join(" · ");
+}
+
+function headerLines(
+  stats,
+  bounds,
+  frameWidth,
+  options,
+  enabled,
+  freshness,
+  sourceStatus,
+  snapshot,
+) {
   const left = colorize("TOKEN LEDGER", TITLE_STYLE, enabled);
   const date = colorize(
     dateLabel(bounds, options.range, options.rollingLabel),
@@ -572,6 +725,23 @@ function headerLines(stats, bounds, frameWidth, options, enabled, freshness) {
   const separator = colorize("·", SECONDARY_STYLE, enabled);
   const join = ` ${separator} `;
   const alignHeader = (line) => fit(` ${line}`, frameWidth);
+  const history = historyScopeLabel(snapshot);
+  const appendHistory = (lines) => history
+    ? [...lines, alignHeader(colorize(history, SECONDARY_STYLE, enabled))]
+    : lines;
+  const appendSnapshotMetadata = (lines) => {
+    const metadata = appendHistory([
+      ...lines,
+      ...(isRollingRange(options.range)
+        ? [alignHeader(snapshotLine(freshness, enabled))]
+        : []),
+      alignHeader(provenanceLine(sourceStatus, enabled)),
+    ]);
+    const warning = incompleteSourceWarning(snapshot);
+    return warning
+      ? [...metadata, alignHeader(colorize(warning, SECONDARY_STYLE, enabled))]
+      : metadata;
+  };
   const fullLine = [
     left,
     date,
@@ -582,9 +752,7 @@ function headerLines(stats, bounds, frameWidth, options, enabled, freshness) {
     metric(stats.projectCount.toLocaleString("en-US"), "PROJECTS"),
   ].join(join);
   if (visibleLength(fullLine) < frameWidth) {
-    const lines = [alignHeader(fullLine)];
-    if (isRollingRange(options.range)) lines.push(alignHeader(snapshotLine(freshness, enabled)));
-    return lines;
+    return appendSnapshotMetadata([alignHeader(fullLine)]);
   }
 
   const compactDate = dateLabel(bounds, options.range, options.rollingLabel)
@@ -607,9 +775,7 @@ function headerLines(stats, bounds, frameWidth, options, enabled, freshness) {
     metric(stats.projectCount.toLocaleString("en-US"), "P"),
   ].join(join);
   if (visibleLength(compactLine) < frameWidth) {
-    const lines = [alignHeader(compactLine)];
-    if (isRollingRange(options.range)) lines.push(alignHeader(snapshotLine(freshness, enabled)));
-    return lines;
+    return appendSnapshotMetadata([alignHeader(compactLine)]);
   }
 
   const minimalTitle = colorize(frameWidth >= 45 ? "LEDGER" : "L", TITLE_STYLE, enabled);
@@ -622,22 +788,21 @@ function headerLines(stats, bounds, frameWidth, options, enabled, freshness) {
     compact(stats.threads),
     compact(stats.projectCount),
   ].join(" ");
-  const lines = [alignHeader(minimalLine)];
-  if (isRollingRange(options.range)) lines.push(alignHeader(snapshotLine(freshness, enabled)));
-  return lines;
+  return appendSnapshotMetadata([alignHeader(minimalLine)]);
 }
 
 export function renderTerminal({
   options,
   snapshot,
   snapshotFreshness,
+  sourceStatus = "unchecked-cache",
   bounds,
   events,
   rows,
   allRows,
 }) {
   const enabled = colorsEnabled(options);
-  const stats = summary(events);
+  const stats = summary(events, allRows);
   const quota = quotaCycleSummary(snapshot, events);
   stats.projectCount = allRows.length;
   const columns = options.width ?? (Number(process.stdout.columns) || 120);
@@ -648,7 +813,16 @@ export function renderTerminal({
   const left = panelLines(rows, allRows, stats.totalTokens, leftWidth, options, enabled);
   const right = sideBySide ? sidebarLines(stats, sideWidth, enabled, options, quota) : null;
   const lines = [
-    ...headerLines(stats, bounds, frameWidth, options, enabled, snapshotFreshness),
+    ...headerLines(
+      stats,
+      bounds,
+      frameWidth,
+      options,
+      enabled,
+      snapshotFreshness,
+      sourceStatus,
+      snapshot,
+    ),
     ...panel(left, right, leftWidth, sideWidth, enabled, options.ascii),
   ];
   if (!sideBySide) {
@@ -688,6 +862,7 @@ export function renderFullscreen({
   options,
   snapshot,
   snapshotFreshness,
+  sourceStatus = "unchecked-cache",
   bounds,
   events,
   rows,
@@ -710,6 +885,7 @@ export function renderFullscreen({
     },
     snapshot,
     snapshotFreshness,
+    sourceStatus,
     bounds,
     events,
     rows,