tokentrace 0.8.0 → 0.8.2

package/CHANGELOG.md

@@ -2,6 +2,32 @@
 
 All notable changes to TokenTrace are documented here.
 
+## [0.8.2] - 2026-05-13
+
+### Fixed
+
+- Codex CLI `token_count` imports now preserve non-cached input tokens and add cached input tokens to processed totals, matching Codex's `input (+ cached)` session summary.
+- Codex parser provenance is bumped to version 3 so previously imported Codex session files are reprocessed on the next scan instead of keeping stale undercounted rows.
+- Generic text-log parsing now recognizes Codex `Token usage: total=... input=... (+ ... cached) output=...` summary lines as structured usage instead of weak text estimates.
+- Overview and evidence token totals now switch to billions at large scales instead of displaying multi-thousand million values.
+- Scan, settings, pricing, and pricing-refresh API writes now reject malformed JSON and avoid JavaScript truthiness coercion for boolean flags.
+- Settings and scan custom folders are trimmed and blank entries are discarded before persistence or scan execution.
+- Pricing manifest imports now ignore invalid, boolean, array, blank, and negative price values instead of coercing them into trusted numeric prices.
+
+## [0.8.1] - 2026-05-13
+
+### Fixed
+
+- Codex CLI imports now read exact `token_count` totals, including cached input and reasoning output tokens, so cleared Codex sessions are no longer undercounted.
+- Claude Code and Codex session artifacts can exceed the generic file-size cap without being skipped during discovery.
+- OpenAI-style, Claude-style, and generic usage parsers now normalize cached, cache-write, and reasoning token fields without double-counting them.
+- Parser-version-aware rescans now reprocess stale imports, and source-file replacement is atomic so a failed replacement cannot delete prior trusted sessions.
+- Unknown-cost cause summaries now assign each interaction to one primary cause instead of overlapping buckets.
+- Pricing, scan, settings, repair, and pricing-refresh APIs now reject malformed JSON with clean 400 responses.
+- Manual pricing saves now reject blank model/provider names and invalid numeric prices instead of silently storing unknown prices.
+- CLI commands now reject unknown flags across scan, pricing refresh, status, doctor, digest, and insights commands.
+- Local release checks now run ProjScan through `projscan@latest`, matching the GitHub security workflow guardrail.
+
 ## [0.8.0] - 2026-05-12
 
 ### Added

package/app/api/prices/refresh/route.ts

@@ -1,13 +1,28 @@
 import { NextResponse } from "next/server";
+import { jsonBooleanFlag, readOptionalJsonObject } from "@/src/lib/api-json";
 import { refreshPricing } from "@/src/lib/pricing-refresh";
 
 export const dynamic = "force-dynamic";
 
+function refreshSource(value: unknown) {
+  if (value == null) return "remote";
+  if (value === "remote" || value === "bundled") return value;
+  return null;
+}
+
 export async function POST(request: Request) {
-  const body = await request.json().catch(() => ({}));
+  const parsed = await readOptionalJsonObject(request);
+  if (!parsed.ok) {
+    return NextResponse.json({ error: parsed.error }, { status: 400 });
+  }
+  const body = parsed.body;
+  const source = refreshSource(body.source);
+  if (!source) {
+    return NextResponse.json({ error: "source must be remote or bundled" }, { status: 400 });
+  }
   const result = await refreshPricing({
-    source: body?.source === "bundled" ? "bundled" : "remote",
-    force: Boolean(body?.force)
+    source,
+    force: jsonBooleanFlag(body.force)
   });
   return NextResponse.json(result);
 }

package/app/api/prices/route.ts

@@ -1,12 +1,24 @@
 import { NextResponse } from "next/server";
+import { readJsonObject } from "@/src/lib/api-json";
 import { getPricingRows, upsertPricing } from "@/src/lib/pricing";
 
 export const dynamic = "force-dynamic";
 
-function nullableNumber(value: unknown) {
-  if (value === "" || value == null) return null;
+function requiredText(value: unknown) {
+  return typeof value === "string" ? value.trim() : "";
+}
+
+function nullablePrice(value: unknown, field: string) {
+  if (value == null) return { ok: true as const, value: null };
+  if (typeof value === "string" && value.trim() === "") return { ok: true as const, value: null };
+  if (typeof value !== "number" && typeof value !== "string") {
+    return { ok: false as const, error: `${field} must be a non-negative number or empty` };
+  }
   const number = Number(value);
-  return Number.isFinite(number) ? number : null;
+  if (!Number.isFinite(number) || number < 0) {
+    return { ok: false as const, error: `${field} must be a non-negative number or empty` };
+  }
+  return { ok: true as const, value: number };
 }
 
 export async function GET() {
@@ -14,20 +26,34 @@ export async function GET() {
 }
 
 export async function POST(request: Request) {
-  const body = await request.json();
-  if (!body.providerId || !body.model) {
+  const parsed = await readJsonObject(request);
+  if (!parsed.ok) {
+    return NextResponse.json({ error: parsed.error }, { status: 400 });
+  }
+  const body = parsed.body;
+  const providerId = requiredText(body.providerId);
+  const model = requiredText(body.model);
+  if (!providerId || !model) {
     return NextResponse.json({ error: "providerId and model are required" }, { status: 400 });
   }
+  const inputTokenPrice = nullablePrice(body.inputTokenPrice, "inputTokenPrice");
+  const outputTokenPrice = nullablePrice(body.outputTokenPrice, "outputTokenPrice");
+  const cachedInputTokenPrice = nullablePrice(body.cachedInputTokenPrice, "cachedInputTokenPrice");
+  const cacheWriteTokenPrice = nullablePrice(body.cacheWriteTokenPrice, "cacheWriteTokenPrice");
+  if (!inputTokenPrice.ok) return NextResponse.json({ error: inputTokenPrice.error }, { status: 400 });
+  if (!outputTokenPrice.ok) return NextResponse.json({ error: outputTokenPrice.error }, { status: 400 });
+  if (!cachedInputTokenPrice.ok) return NextResponse.json({ error: cachedInputTokenPrice.error }, { status: 400 });
+  if (!cacheWriteTokenPrice.ok) return NextResponse.json({ error: cacheWriteTokenPrice.error }, { status: 400 });
 
   const id = upsertPricing({
-    providerId: String(body.providerId),
-    providerName: body.providerName ? String(body.providerName) : undefined,
-    model: String(body.model),
-    inputTokenPrice: nullableNumber(body.inputTokenPrice),
-    outputTokenPrice: nullableNumber(body.outputTokenPrice),
-    cachedInputTokenPrice: nullableNumber(body.cachedInputTokenPrice),
-    cacheWriteTokenPrice: nullableNumber(body.cacheWriteTokenPrice),
-    currency: body.currency ? String(body.currency) : "USD"
+    providerId,
+    providerName: requiredText(body.providerName) || undefined,
+    model,
+    inputTokenPrice: inputTokenPrice.value,
+    outputTokenPrice: outputTokenPrice.value,
+    cachedInputTokenPrice: cachedInputTokenPrice.value,
+    cacheWriteTokenPrice: cacheWriteTokenPrice.value,
+    currency: requiredText(body.currency) || "USD"
   });
 
   return NextResponse.json({ id, costsRecalculated: true });

package/app/api/repair-items/route.ts

@@ -5,6 +5,7 @@ import {
   saveUnknownCostReview,
   type UnknownCostRepairStatus
 } from "@/src/lib/unknown-cost-repair";
+import { readJsonObject } from "@/src/lib/api-json";
 
 export const dynamic = "force-dynamic";
 
@@ -34,7 +35,11 @@ export async function GET() {
 }
 
 export async function PUT(request: Request) {
-  const body = await request.json();
+  const parsed = await readJsonObject(request);
+  if (!parsed.ok) {
+    return NextResponse.json({ error: parsed.error }, { status: 400 });
+  }
+  const body = parsed.body;
   const key = text(body.key, 1000);
   const status = reviewState(body.status ?? body.state);
 

package/app/api/scan/route.ts

@@ -1,15 +1,27 @@
 import { NextResponse } from "next/server";
 import { runScan } from "@/src/ingestion/scan";
+import { jsonBooleanFlag, readOptionalJsonObject } from "@/src/lib/api-json";
 
 export const dynamic = "force-dynamic";
 
+function stringList(value: unknown) {
+  return Array.isArray(value)
+    ? value
+        .filter((folder): folder is string => typeof folder === "string")
+        .map((folder) => folder.trim())
+        .filter(Boolean)
+    : undefined;
+}
+
 export async function POST(request: Request) {
-  const body = await request.json().catch(() => ({}));
+  const parsed = await readOptionalJsonObject(request);
+  if (!parsed.ok) {
+    return NextResponse.json({ error: parsed.error }, { status: 400 });
+  }
+  const body = parsed.body;
   const result = await runScan({
-    folders: Array.isArray(body.folders)
-      ? body.folders.filter((folder: unknown): folder is string => typeof folder === "string")
-      : undefined,
-    force: Boolean(body.force)
+    folders: stringList(body.folders),
+    force: jsonBooleanFlag(body.force)
   });
   return NextResponse.json(result);
 }

package/app/api/settings/route.ts

@@ -1,6 +1,7 @@
 import { NextResponse } from "next/server";
 import { getDatabasePath } from "@/src/db/client";
 import { getAppSettings, normalizeUsageGuardrails, saveAppSettings } from "@/src/db/settings";
+import { jsonBooleanFlag, readJsonObject } from "@/src/lib/api-json";
 
 export const dynamic = "force-dynamic";
 
@@ -12,13 +13,20 @@ export async function GET() {
 }
 
 export async function PUT(request: Request) {
-  const body = await request.json();
+  const parsed = await readJsonObject(request);
+  if (!parsed.ok) {
+    return NextResponse.json({ error: parsed.error }, { status: 400 });
+  }
+  const body = parsed.body;
   const customFolders = Array.isArray(body.customFolders)
-    ? body.customFolders.filter((folder: unknown): folder is string => typeof folder === "string")
+    ? body.customFolders
+        .filter((folder: unknown): folder is string => typeof folder === "string")
+        .map((folder) => folder.trim())
+        .filter(Boolean)
     : [];
   const saved = saveAppSettings({
     customFolders,
-    storeRawMessageContent: Boolean(body.storeRawMessageContent),
+    storeRawMessageContent: jsonBooleanFlag(body.storeRawMessageContent),
     usageGuardrails: normalizeUsageGuardrails(body.usageGuardrails)
   });
 

package/app/evidence/page.tsx

@@ -6,7 +6,7 @@ import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/com
 import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
 import { DataValue, FieldLabel, MonoText, PageHeader } from "@/components/ui/typography";
 import { buildEvidenceTrail, parseEvidenceMetric } from "@/src/lib/evidence-trail";
-import { formatCurrency, formatTokens, percent } from "@/src/lib/format";
+import { formatCurrency, formatExactTokens, percent } from "@/src/lib/format";
 
 export const dynamic = "force-dynamic";
 
@@ -57,7 +57,7 @@ export default async function EvidencePage({ searchParams }: EvidencePageProps)
           <div className="grid divide-y border-t sm:grid-cols-2 sm:divide-x sm:divide-y-0 lg:grid-cols-5">
             <div className="p-3">
               <FieldLabel>Tokens</FieldLabel>
-              <DataValue className="mt-1" size="md">{formatTokens(trail.totals.tokens)}</DataValue>
+              <DataValue className="mt-1" size="md">{formatExactTokens(trail.totals.tokens)}</DataValue>
             </div>
             <div className="p-3">
               <FieldLabel>Cost</FieldLabel>
@@ -111,7 +111,7 @@ export default async function EvidencePage({ searchParams }: EvidencePageProps)
                         {session.tool} / {session.provider} / {session.project}
                       </div>
                     </TableCell>
-                    <TableCell>{formatTokens(session.totalTokens)}</TableCell>
+                    <TableCell>{formatExactTokens(session.totalTokens)}</TableCell>
                     <TableCell>{formatCurrency(session.cost)}</TableCell>
                     <TableCell className="max-w-80">
                       <Link href={session.sourceHref} title={session.sourceFile}>

package/app/page.tsx

@@ -15,7 +15,7 @@ import { getDefaultSearchRoots } from "@/src/ingestion/discovery";
 import { buildFirstRunStatus, type FirstRunStatus } from "@/src/lib/first-run-status";
 import { buildUnknownCostRepairWorkbench } from "@/src/lib/unknown-cost-repair";
 import { resolveDateRange } from "@/src/lib/date-range";
-import { formatCurrency, formatTokens, percent } from "@/src/lib/format";
+import { formatCurrency, formatSignedTokens, formatTokens, percent } from "@/src/lib/format";
 import { cn } from "@/src/lib/utils";
 import type { UsageGuardrailMetric } from "@/src/lib/usage-guardrails";
 
@@ -74,16 +74,16 @@ function MetricCard({
   );
 }
 
-function formatSignedNumber(value: number) {
-  if (value === 0) return "0";
-  return `${value > 0 ? "+" : "-"}${Math.abs(value).toLocaleString()}`;
-}
-
 function formatSignedCurrency(value: number) {
   if (value === 0) return "$0.00";
   return `${value > 0 ? "+" : "-"}${formatCurrency(Math.abs(value))}`;
 }
 
+function formatSignedNumber(value: number) {
+  if (value === 0) return "0";
+  return `${value > 0 ? "+" : "-"}${Math.abs(value).toLocaleString()}`;
+}
+
 function formatPercentValue(value: number | null) {
   if (value == null) return "new";
   if (value === 0) return "flat";
@@ -319,7 +319,7 @@ export default async function OverviewPage({ searchParams }: OverviewPageProps)
             <DeltaMetric
               label="Tokens"
               value={formatTokens(data.comparison.current.totalTokens)}
-              delta={formatSignedNumber(data.comparison.delta.totalTokens)}
+              delta={formatSignedTokens(data.comparison.delta.totalTokens)}
               percentValue={data.comparison.delta.totalTokensPercent}
               previous={formatTokens(data.comparison.previous.totalTokens)}
             />

package/dist/runtime/db-seed.mjs

@@ -1193,8 +1193,16 @@ var default_model_prices_default = {
 // src/lib/pricing-manifest.ts
 function nullableNumber(value) {
   if (value == null) return null;
-  const number = Number(value);
-  return Number.isFinite(number) ? number : null;
+  if (typeof value === "number") {
+    return Number.isFinite(value) && value >= 0 ? value : null;
+  }
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    if (!trimmed) return null;
+    const number = Number(trimmed);
+    return Number.isFinite(number) && number >= 0 ? number : null;
+  }
+  return null;
 }
 function validModel(value) {
   if (!value || typeof value !== "object") return null;