tokentrace 0.8.1 → 0.8.3

package/CHANGELOG.md

@@ -2,6 +2,25 @@
 
 All notable changes to TokenTrace are documented here.
 
+## [0.8.3] - 2026-05-13
+
+### Fixed
+
+- Codex CLI `token_count` JSON imports now subtract cached input from non-cached input and keep cached input separate, so processed Codex totals count cache exactly once instead of double-counting it.
+- Codex parser provenance is bumped to version 4 so previously imported version-3 Codex session rows are reprocessed on the next scan with corrected token and cost totals.
+
+## [0.8.2] - 2026-05-13
+
+### Fixed
+
+- Codex CLI `token_count` imports now preserve non-cached input tokens and add cached input tokens to processed totals, matching Codex's `input (+ cached)` session summary.
+- Codex parser provenance is bumped to version 3 so previously imported Codex session files are reprocessed on the next scan instead of keeping stale undercounted rows.
+- Generic text-log parsing now recognizes Codex `Token usage: total=... input=... (+ ... cached) output=...` summary lines as structured usage instead of weak text estimates.
+- Overview and evidence token totals now switch to billions at large scales instead of displaying multi-thousand million values.
+- Scan, settings, pricing, and pricing-refresh API writes now reject malformed JSON and avoid JavaScript truthiness coercion for boolean flags.
+- Settings and scan custom folders are trimmed and blank entries are discarded before persistence or scan execution.
+- Pricing manifest imports now ignore invalid, boolean, array, blank, and negative price values instead of coercing them into trusted numeric prices.
+
 ## [0.8.1] - 2026-05-13
 
 ### Fixed

package/app/api/prices/refresh/route.ts

@@ -1,18 +1,28 @@
 import { NextResponse } from "next/server";
-import { readJsonObject } from "@/src/lib/api-json";
+import { jsonBooleanFlag, readOptionalJsonObject } from "@/src/lib/api-json";
 import { refreshPricing } from "@/src/lib/pricing-refresh";
 
 export const dynamic = "force-dynamic";
 
+function refreshSource(value: unknown) {
+  if (value == null) return "remote";
+  if (value === "remote" || value === "bundled") return value;
+  return null;
+}
+
 export async function POST(request: Request) {
-  const parsed = await readJsonObject(request);
+  const parsed = await readOptionalJsonObject(request);
   if (!parsed.ok) {
     return NextResponse.json({ error: parsed.error }, { status: 400 });
   }
   const body = parsed.body;
+  const source = refreshSource(body.source);
+  if (!source) {
+    return NextResponse.json({ error: "source must be remote or bundled" }, { status: 400 });
+  }
   const result = await refreshPricing({
-    source: body?.source === "bundled" ? "bundled" : "remote",
-    force: Boolean(body?.force)
+    source,
+    force: jsonBooleanFlag(body.force)
   });
   return NextResponse.json(result);
 }

package/app/api/prices/route.ts

@@ -11,6 +11,9 @@ function requiredText(value: unknown) {
 function nullablePrice(value: unknown, field: string) {
   if (value == null) return { ok: true as const, value: null };
   if (typeof value === "string" && value.trim() === "") return { ok: true as const, value: null };
+  if (typeof value !== "number" && typeof value !== "string") {
+    return { ok: false as const, error: `${field} must be a non-negative number or empty` };
+  }
   const number = Number(value);
   if (!Number.isFinite(number) || number < 0) {
     return { ok: false as const, error: `${field} must be a non-negative number or empty` };
@@ -42,7 +45,7 @@ export async function POST(request: Request) {
   if (!cachedInputTokenPrice.ok) return NextResponse.json({ error: cachedInputTokenPrice.error }, { status: 400 });
   if (!cacheWriteTokenPrice.ok) return NextResponse.json({ error: cacheWriteTokenPrice.error }, { status: 400 });
 
-  const id = upsertPricing({
+  const result = upsertPricing({
     providerId,
     providerName: requiredText(body.providerName) || undefined,
     model,
@@ -53,5 +56,12 @@ export async function POST(request: Request) {
     currency: requiredText(body.currency) || "USD"
   });
 
-  return NextResponse.json({ id, costsRecalculated: true });
+  return NextResponse.json({
+    id: result.id,
+    costsRecalculated: result.interactionsUpdated,
+    interactionsChecked: result.interactionsChecked,
+    unknownCostInteractions: result.unknownCostInteractions,
+    modelAliasesUpdated: result.modelsUpdated,
+    resolvedRepairItems: result.resolvedRepairItems
+  });
 }

package/app/api/scan/route.ts

@@ -1,20 +1,27 @@
 import { NextResponse } from "next/server";
 import { runScan } from "@/src/ingestion/scan";
-import { readJsonObject } from "@/src/lib/api-json";
+import { jsonBooleanFlag, readOptionalJsonObject } from "@/src/lib/api-json";
 
 export const dynamic = "force-dynamic";
 
+function stringList(value: unknown) {
+  return Array.isArray(value)
+    ? value
+        .filter((folder): folder is string => typeof folder === "string")
+        .map((folder) => folder.trim())
+        .filter(Boolean)
+    : undefined;
+}
+
 export async function POST(request: Request) {
-  const parsed = await readJsonObject(request);
+  const parsed = await readOptionalJsonObject(request);
   if (!parsed.ok) {
     return NextResponse.json({ error: parsed.error }, { status: 400 });
   }
   const body = parsed.body;
   const result = await runScan({
-    folders: Array.isArray(body.folders)
-      ? body.folders.filter((folder: unknown): folder is string => typeof folder === "string")
-      : undefined,
-    force: Boolean(body.force)
+    folders: stringList(body.folders),
+    force: jsonBooleanFlag(body.force)
   });
   return NextResponse.json(result);
 }

package/app/api/settings/route.ts

@@ -1,7 +1,7 @@
 import { NextResponse } from "next/server";
 import { getDatabasePath } from "@/src/db/client";
 import { getAppSettings, normalizeUsageGuardrails, saveAppSettings } from "@/src/db/settings";
-import { readJsonObject } from "@/src/lib/api-json";
+import { jsonBooleanFlag, readJsonObject } from "@/src/lib/api-json";
 
 export const dynamic = "force-dynamic";
 
@@ -19,11 +19,14 @@ export async function PUT(request: Request) {
   }
   const body = parsed.body;
   const customFolders = Array.isArray(body.customFolders)
-    ? body.customFolders.filter((folder: unknown): folder is string => typeof folder === "string")
+    ? body.customFolders
+        .filter((folder: unknown): folder is string => typeof folder === "string")
+        .map((folder) => folder.trim())
+        .filter(Boolean)
     : [];
   const saved = saveAppSettings({
     customFolders,
-    storeRawMessageContent: Boolean(body.storeRawMessageContent),
+    storeRawMessageContent: jsonBooleanFlag(body.storeRawMessageContent),
     usageGuardrails: normalizeUsageGuardrails(body.usageGuardrails)
   });
 

package/app/evidence/page.tsx

@@ -1,12 +1,14 @@
 import Link from "next/link";
 import { ArrowLeft, ArrowRight } from "lucide-react";
+import { PeriodFilter } from "@/components/period-filter";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
 import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
 import { DataValue, FieldLabel, MonoText, PageHeader } from "@/components/ui/typography";
+import { dateRangeQueryParams, mergeHrefParams, resolveDateRange } from "@/src/lib/date-range";
 import { buildEvidenceTrail, parseEvidenceMetric } from "@/src/lib/evidence-trail";
-import { formatCurrency, formatTokens, percent } from "@/src/lib/format";
+import { formatCurrency, formatExactTokens, percent } from "@/src/lib/format";
 
 export const dynamic = "force-dynamic";
 
@@ -29,7 +31,14 @@ function parserStatusVariant(value: string | null) {
 
 export default async function EvidencePage({ searchParams }: EvidencePageProps) {
   const params = (await searchParams) ?? {};
-  const trail = buildEvidenceTrail({ metric: parseEvidenceMetric(params?.metric) });
+  const range = resolveDateRange(params);
+  const metric = parseEvidenceMetric(params?.metric);
+  const trail = buildEvidenceTrail({ metric, filters: range.filters });
+  const rangeLinkParams = dateRangeQueryParams(range);
+  const overviewHref = mergeHrefParams("/", rangeLinkParams);
+  const currentEvidenceHref = mergeHrefParams(`/evidence?metric=${trail.metric}`, rangeLinkParams);
+  const pricingReturnParams = { returnTo: currentEvidenceHref };
+  const confidenceTotal = Math.max(1, trail.confidence.exact + trail.confidence.estimated + trail.confidence.unknown);
 
   return (
     <div className="space-y-6">
@@ -38,7 +47,7 @@ export default async function EvidencePage({ searchParams }: EvidencePageProps)
         description={trail.description}
         actions={
           <Button asChild variant="outline" size="sm">
-            <Link href="/">
+            <Link href={overviewHref}>
               <ArrowLeft className="h-4 w-4" />
               Overview
             </Link>
@@ -46,6 +55,8 @@ export default async function EvidencePage({ searchParams }: EvidencePageProps)
         }
       />
 
+      <PeriodFilter range={range} />
+
       <Card>
         <CardHeader>
           <CardTitle>Metric Totals</CardTitle>
@@ -57,7 +68,7 @@ export default async function EvidencePage({ searchParams }: EvidencePageProps)
           <div className="grid divide-y border-t sm:grid-cols-2 sm:divide-x sm:divide-y-0 lg:grid-cols-5">
             <div className="p-3">
               <FieldLabel>Tokens</FieldLabel>
-              <DataValue className="mt-1" size="md">{formatTokens(trail.totals.tokens)}</DataValue>
+              <DataValue className="mt-1" size="md">{formatExactTokens(trail.totals.tokens)}</DataValue>
             </div>
             <div className="p-3">
               <FieldLabel>Cost</FieldLabel>
@@ -79,6 +90,84 @@ export default async function EvidencePage({ searchParams }: EvidencePageProps)
         </CardContent>
       </Card>
 
+      <div className="grid gap-4 xl:grid-cols-[minmax(0,0.75fr)_minmax(0,1.25fr)]">
+        <Card>
+          <CardHeader>
+            <CardTitle>Confidence Split</CardTitle>
+            <CardDescription>Interaction-level token confidence for this metric and period.</CardDescription>
+          </CardHeader>
+          <CardContent className="space-y-3">
+            {[
+              { label: "Exact", value: trail.confidence.exact, variant: "success" as const },
+              { label: "Estimated", value: trail.confidence.estimated, variant: "secondary" as const },
+              { label: "Unknown", value: trail.confidence.unknown, variant: "warning" as const }
+            ].map((item) => (
+              <div key={item.label} className="grid grid-cols-[5.5rem_minmax(0,1fr)_4rem] items-center gap-3 text-sm">
+                <Badge variant={item.variant}>{item.label}</Badge>
+                <div className="h-2 overflow-hidden rounded-full bg-muted">
+                  <div className="h-full rounded-full bg-primary" style={{ width: `${(item.value / confidenceTotal) * 100}%` }} />
+                </div>
+                <div className="text-right tabular-nums text-muted-foreground">{item.value.toLocaleString()}</div>
+              </div>
+            ))}
+          </CardContent>
+        </Card>
+
+        <Card>
+          <CardHeader>
+            <CardTitle>Top Source Files</CardTitle>
+            <CardDescription>Largest contributing local files for the same metric definition.</CardDescription>
+          </CardHeader>
+          <CardContent className="table-scroll">
+            <Table>
+              <TableHeader>
+                <TableRow>
+                  <TableHead>Source</TableHead>
+                  <TableHead>Tokens</TableHead>
+                  <TableHead>Interactions</TableHead>
+                  <TableHead>Unknown cost</TableHead>
+                  <TableHead>Actions</TableHead>
+                </TableRow>
+              </TableHeader>
+              <TableBody>
+                {trail.sourceFiles.length ? (
+                  trail.sourceFiles.map((source) => (
+                    <TableRow key={source.sourceFile}>
+                      <TableCell className="max-w-96">
+                        <Link href={mergeHrefParams(source.sourceHref, rangeLinkParams)} title={source.sourceFile}>
+                          <MonoText className="block truncate text-muted-foreground underline-offset-4 hover:underline">
+                            {source.sourceFile}
+                          </MonoText>
+                        </Link>
+                      </TableCell>
+                      <TableCell>{formatExactTokens(source.tokens)}</TableCell>
+                      <TableCell>{source.interactions.toLocaleString()}</TableCell>
+                      <TableCell>{source.unknownCostInteractions.toLocaleString()}</TableCell>
+                      <TableCell>
+                        <div className="flex flex-wrap gap-2">
+                          <Link href={mergeHrefParams(source.sourceHref, rangeLinkParams)} className="font-medium text-primary underline-offset-4 hover:underline">
+                            Sessions
+                          </Link>
+                          <Link href={mergeHrefParams(source.parserHref, rangeLinkParams)} className="font-medium text-muted-foreground underline-offset-4 hover:underline">
+                            Parser
+                          </Link>
+                        </div>
+                      </TableCell>
+                    </TableRow>
+                  ))
+                ) : (
+                  <TableRow>
+                    <TableCell colSpan={5} className="h-20 text-center text-muted-foreground">
+                      No source-file evidence is available for this metric yet.
+                    </TableCell>
+                  </TableRow>
+                )}
+              </TableBody>
+            </Table>
+          </CardContent>
+        </Card>
+      </div>
+
       <Card>
         <CardHeader>
           <CardTitle>Session, Source, Parser, And Pricing Evidence</CardTitle>
@@ -104,17 +193,17 @@ export default async function EvidencePage({ searchParams }: EvidencePageProps)
                 trail.sessions.map((session) => (
                   <TableRow key={session.id}>
                     <TableCell className="min-w-64">
-                      <Link href={session.sessionHref} className="font-medium text-primary underline-offset-4 hover:underline">
+                      <Link href={mergeHrefParams(session.sessionHref, rangeLinkParams)} className="font-medium text-primary underline-offset-4 hover:underline">
                         {session.title}
                       </Link>
                       <div className="mt-1 text-xs text-muted-foreground">
                         {session.tool} / {session.provider} / {session.project}
                       </div>
                     </TableCell>
-                    <TableCell>{formatTokens(session.totalTokens)}</TableCell>
+                    <TableCell>{formatExactTokens(session.totalTokens)}</TableCell>
                     <TableCell>{formatCurrency(session.cost)}</TableCell>
                     <TableCell className="max-w-80">
-                      <Link href={session.sourceHref} title={session.sourceFile}>
+                      <Link href={mergeHrefParams(session.sourceHref, rangeLinkParams)} title={session.sourceFile}>
                         <MonoText className="block truncate text-muted-foreground underline-offset-4 hover:underline">
                           {session.sourceFile}
                         </MonoText>
@@ -125,7 +214,7 @@ export default async function EvidencePage({ searchParams }: EvidencePageProps)
                         <Badge variant={parserStatusVariant(session.parserStatus)}>
                           {session.parserStatus ?? "not scanned"}
                         </Badge>
-                        <Link href={session.parserHref} className="text-xs font-medium text-primary underline-offset-4 hover:underline">
+                        <Link href={mergeHrefParams(session.parserHref, rangeLinkParams)} className="text-xs font-medium text-primary underline-offset-4 hover:underline">
                           Parser <ArrowRight className="inline h-3.5 w-3.5" />
                         </Link>
                       </div>
@@ -135,7 +224,7 @@ export default async function EvidencePage({ searchParams }: EvidencePageProps)
                     </TableCell>
                     <TableCell className="min-w-44">
                       {session.pricingHref ? (
-                        <Link href={session.pricingHref} className="font-medium text-primary underline-offset-4 hover:underline">
+                        <Link href={mergeHrefParams(session.pricingHref, pricingReturnParams)} className="font-medium text-primary underline-offset-4 hover:underline">
                           {session.model}
                         </Link>
                       ) : (