tokentrace 0.16.0 → 0.17.0

package/CHANGELOG.md

@@ -4,6 +4,64 @@ All notable changes to TokenTrace are documented here.
 
 ## Unreleased
 
+## [0.17.0] - 2026-05-23
+
+### Performance
+
+- **Runtime SQLite pragmas tuned for analytics.** `src/db/client.ts` now
+  re-applies `journal_mode=WAL` on the live connection and sets
+  `synchronous=NORMAL`, `temp_store=MEMORY`, `cache_size=64MB`, and
+  `mmap_size=256MB`. Previously only `busy_timeout` and `foreign_keys`
+  were set on the runtime connection.
+- **Prepared-statement cache.** `src/db/prepared.ts` adds a tiny
+  `prepareCached(sql)` helper keyed by SQL string. The hot analytics,
+  unknown-cost-repair, scheduled-scan, and ingestion helpers now skip
+  the parse-and-plan cost on repeat queries.
+- **Overview page parallelization.** The independent sub-queries in
+  `getOverviewData` (analytics, accounting invariants, scan diff,
+  default search roots, repair workbench) now run through `Promise.all`,
+  so the async filesystem walk overlaps with the serialized SQLite
+  reads.
+- **Render-scoped overview memo.** `getOverviewPageData` is wrapped in
+  `React.cache` so any future composition that calls it twice within a
+  single server render tree pays the cost once.
+- **Lazy-loaded Recharts.** `TrendSection` and `RankBarChart` are now
+  loaded via `next/dynamic({ ssr: false })` on the overview, projects,
+  tools, and models routes, splitting the ~80KB Recharts bundle out of
+  the initial JS payload. `ChartSkeleton` keeps the chart slot from
+  collapsing during client hydration.
+- **Streaming overview with two `<Suspense>` boundaries.** The overview
+  page now splits into two cache-wrapped fetchers — `getOverviewPrimaryData`
+  (analytics-driven: pulse, summary, trend, trust, guardrails,
+  recommendations, mix) and `getOverviewRepairData` (workbench-driven:
+  review status, repair items). Each renders inside its own `<Suspense>`
+  so the page shell and period filter paint immediately, the primary
+  analytics block streams in next, and the repair lane streams in
+  independently when the workbench query resolves.
+- **`tokentrace doctor --timings`.** New flag force-enables analytics
+  timing capture and emits the analytics timing report (slow queries,
+  threshold, sample list). Combine with `--json` for machine-readable
+  output. Useful for before/after measurement of performance changes
+  since `TOKENTRACE_ANALYTICS_TIMING` is off by default in production.
+- **Scan ingestion throughput.** Adds a (path, size, mtime)-keyed
+  file-hash cache so rescans of unchanged files skip the `fs.readFile`
+  + SHA-256 step entirely. The hot scan-side `INSERT INTO scan_files`,
+  `INSERT INTO scan_runs`, and `hasImportedFile` lookups also route
+  through `prepareCached`.
+- **Next bundle optimizations.** Enables `optimizePackageImports` for
+  `lucide-react` (~37 import sites) and `recharts` so Next 16 transforms
+  named imports into per-symbol imports and prunes unused-symbol weight
+  from the client bundle. Adds an opt-in `@next/bundle-analyzer`
+  integration gated on `ANALYZE=true` for follow-up audits.
+- **Hot-path bug-hunt fixes.** `source-catalog.summarizeSourceCoverage`
+  was O(rows × entries); now uses a pre-built Map for O(1) parser tier
+  lookups. `ingestion/discovery.getDefaultSearchRoots` parallelizes its
+  `fs.access` checks via `Promise.all` instead of awaiting them
+  sequentially. `ingestion/persist.findProjectRoot` memoizes the
+  resolved project root by start directory, eliminating duplicate
+  filesystem walks when many imported sessions share the same source
+  directory.
+
 ## [0.16.0] - 2026-05-23
 
 ### Added

package/app/models/page.tsx

@@ -3,7 +3,7 @@ import { EmptyState } from "@/components/empty-state";
 import { ScanNowButton } from "@/components/scan-now-button";
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
 import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
-import { RankBarChart } from "@/components/charts/rank-bar-chart";
+import { RankBarChart } from "@/components/charts/rank-bar-chart-lazy";
 import { PageHeader } from "@/components/ui/typography";
 import { getAnalyticsData } from "@/src/lib/analytics";
 import { formatCurrency, formatTokens } from "@/src/lib/format";

package/app/page.tsx

@@ -1,12 +1,17 @@
+import { Suspense } from "react";
 import Link from "next/link";
 import { ArrowRight } from "lucide-react";
-import { TrendSection } from "@/components/charts/trend-section";
+import { TrendSection } from "@/components/charts/trend-section-lazy";
 import { OverviewCurrentMixPanel } from "@/components/overview/current-mix-panel";
 import { DataConfidenceStrip } from "@/components/overview/data-confidence-strip";
 import { FirstRunPanel } from "@/components/overview/first-run-panel";
 import { UsageGuardrailsPanel } from "@/components/overview/guardrails-panel";
 import { OverviewRecommendationsCard } from "@/components/overview/recommendations-card";
 import { OverviewReviewStatusStrip } from "@/components/overview/review-status-strip";
+import {
+  OverviewPrimarySkeleton,
+  OverviewRepairSkeleton
+} from "@/components/overview/section-skeletons";
 import { CostSessionsCard, TokenAccountingCard } from "@/components/overview/summary-cards";
 import { TopRepairItemsStrip } from "@/components/overview/top-repair-items-strip";
 import { OverviewTrustFooter } from "@/components/overview/trust-footer";
@@ -14,9 +19,9 @@ import { UsagePulsePanel } from "@/components/overview/usage-pulse-panel";
 import { PeriodFilter } from "@/components/period-filter";
 import { Button } from "@/components/ui/button";
 import { PageHeader } from "@/components/ui/typography";
-import { resolveDateRange } from "@/src/lib/date-range";
+import { mergeHrefParams, type ResolvedDateRange, resolveDateRange } from "@/src/lib/date-range";
 import { runDueScheduledScan } from "@/src/lib/scheduled-scan";
-import { getOverviewPageData } from "@/src/lib/overview-data";
+import { getOverviewPrimaryData, getOverviewRepairData } from "@/src/lib/overview-data";
 
 export const dynamic = "force-dynamic";
 
@@ -24,51 +29,24 @@ type OverviewPageProps = {
   searchParams?: Promise<Record<string, string | string[] | undefined>>;
 };
 
-export default async function OverviewPage({ searchParams }: OverviewPageProps) {
-  void runDueScheduledScan().catch(() => undefined);
-  const params = (await searchParams) ?? {};
-  const range = resolveDateRange(params);
-  const overview = await getOverviewPageData(range);
+async function OverviewPrimarySection({ range }: { range: ResolvedDateRange }) {
   const {
     data,
     trust,
-    accountingReport,
-    postSessionReview,
-    rangeLinkParams,
     evidenceLinks,
-    doctorReport,
-    repairWorkbench,
-    repairFocusHref,
-    unknownCostEvidenceHref,
     firstRunStatus,
     summary,
-    trendDefaultWindow
-  } = overview;
+    trendDefaultWindow,
+    unknownCostEvidenceHref,
+    rangeLinkParams
+  } = await getOverviewPrimaryData(range);
+  const repairFocusHref = mergeHrefParams("/repair", rangeLinkParams);
 
   return (
     <div className="space-y-8">
-      <PageHeader
-        title="Overview"
-        description="Local token, cost, model, and session analytics across AI CLI tools."
-        actions={
-          <Button asChild>
-            <Link href="/settings#scan-controls">
-              Configure scan <ArrowRight className="h-4 w-4" />
-            </Link>
-          </Button>
-        }
-      />
-
-      <PeriodFilter range={range} />
-
-      {summary.interactions === 0 ? (
-        <FirstRunPanel status={firstRunStatus} />
-      ) : null}
-
+      {summary.interactions === 0 ? <FirstRunPanel status={firstRunStatus} /> : null}
       <UsagePulsePanel comparison={data.comparison} />
-
       <DataConfidenceStrip confidence={data.dataConfidence} />
-
       <div className="grid gap-3 md:grid-cols-2 xl:grid-cols-6">
         <TokenAccountingCard
           summary={summary}
@@ -84,11 +62,38 @@ export default async function OverviewPage({ searchParams }: OverviewPageProps)
           sessionsHref={evidenceLinks.sessions}
         />
       </div>
-
       <OverviewTrustFooter health={trust.health} pricedModelCount={trust.pricedModelCount} />
-
       <TrendSection data={data.trends} defaultWindow={trendDefaultWindow} />
+      <UsageGuardrailsPanel progress={data.usageGuardrails} />
+      <OverviewRecommendationsCard recommendations={data.recommendations} />
+      <OverviewCurrentMixPanel
+        tools={data.tools}
+        mostUsedTool={summary.mostUsedTool}
+        mostUsedModel={summary.mostUsedModel}
+      />
+    </div>
+  );
+}
+
+async function OverviewRepairSection({ range }: { range: ResolvedDateRange }) {
+  const {
+    accountingReport,
+    postSessionReview,
+    doctorReport,
+    repairWorkbench,
+    repairFocusHref,
+    evidenceLinks,
+    summary,
+    trust,
+    rangeLinkParams
+  } = await getOverviewRepairData(range);
 
+  if (summary.interactions === 0 && repairWorkbench.groups.length === 0) {
+    return null;
+  }
+
+  return (
+    <div className="space-y-8">
       {summary.interactions > 0 ? (
         <OverviewReviewStatusStrip
           report={doctorReport}
@@ -102,7 +107,6 @@ export default async function OverviewPage({ searchParams }: OverviewPageProps)
           cachedEvidenceHref={evidenceLinks["cached-tokens"]}
         />
       ) : null}
-
       {repairWorkbench.groups.length ? (
         <TopRepairItemsStrip
           groups={repairWorkbench.groups}
@@ -110,17 +114,38 @@ export default async function OverviewPage({ searchParams }: OverviewPageProps)
           rangeLinkParams={rangeLinkParams}
         />
       ) : null}
+    </div>
+  );
+}
 
-      <UsageGuardrailsPanel progress={data.usageGuardrails} />
-
-      <OverviewRecommendationsCard recommendations={data.recommendations} />
+export default async function OverviewPage({ searchParams }: OverviewPageProps) {
+  void runDueScheduledScan().catch(() => undefined);
+  const params = (await searchParams) ?? {};
+  const range = resolveDateRange(params);
 
-      <OverviewCurrentMixPanel
-        tools={data.tools}
-        mostUsedTool={summary.mostUsedTool}
-        mostUsedModel={summary.mostUsedModel}
+  return (
+    <div className="space-y-8">
+      <PageHeader
+        title="Overview"
+        description="Local token, cost, model, and session analytics across AI CLI tools."
+        actions={
+          <Button asChild>
+            <Link href="/settings#scan-controls">
+              Configure scan <ArrowRight className="h-4 w-4" />
+            </Link>
+          </Button>
+        }
       />
 
+      <PeriodFilter range={range} />
+
+      <Suspense fallback={<OverviewPrimarySkeleton />}>
+        <OverviewPrimarySection range={range} />
+      </Suspense>
+
+      <Suspense fallback={<OverviewRepairSkeleton />}>
+        <OverviewRepairSection range={range} />
+      </Suspense>
     </div>
   );
 }

package/app/projects/page.tsx

@@ -1,5 +1,5 @@
 import Link from "next/link";
-import { RankBarChart } from "@/components/charts/rank-bar-chart";
+import { RankBarChart } from "@/components/charts/rank-bar-chart-lazy";
 import { TrendChart } from "@/components/charts/trend-chart";
 import { EmptyState } from "@/components/empty-state";
 import { ScanNowButton } from "@/components/scan-now-button";

package/app/tools/page.tsx

@@ -1,4 +1,4 @@
-import { RankBarChart } from "@/components/charts/rank-bar-chart";
+import { RankBarChart } from "@/components/charts/rank-bar-chart-lazy";
 import { EmptyState } from "@/components/empty-state";
 import { ScanNowButton } from "@/components/scan-now-button";
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";

package/components/charts/rank-bar-chart-lazy.tsx

@@ -0,0 +1,9 @@
+"use client";
+
+import dynamic from "next/dynamic";
+import { ChartSkeleton } from "@/components/charts/skeleton";
+
+export const RankBarChart = dynamic(
+  () => import("./rank-bar-chart").then((m) => m.RankBarChart),
+  { ssr: false, loading: () => <ChartSkeleton heightClass="h-60" label="Loading chart…" /> }
+);

package/components/charts/skeleton.tsx

@@ -0,0 +1,16 @@
+type ChartSkeletonProps = {
+  heightClass?: string;
+  label?: string;
+};
+
+export function ChartSkeleton({ heightClass = "h-64", label = "Loading chart…" }: ChartSkeletonProps) {
+  return (
+    <div
+      className={`flex items-center justify-center rounded-md border border-dashed border-muted-foreground/30 bg-muted/30 text-xs text-muted-foreground ${heightClass}`}
+      role="status"
+      aria-live="polite"
+    >
+      {label}
+    </div>
+  );
+}

package/components/charts/trend-section-lazy.tsx

@@ -0,0 +1,9 @@
+"use client";
+
+import dynamic from "next/dynamic";
+import { ChartSkeleton } from "@/components/charts/skeleton";
+
+export const TrendSection = dynamic(
+  () => import("./trend-section").then((m) => m.TrendSection),
+  { ssr: false, loading: () => <ChartSkeleton heightClass="h-72" label="Loading trend chart…" /> }
+);

package/components/overview/current-mix-panel.tsx

@@ -1,5 +1,5 @@
 import Link from "next/link";
-import { RankBarChart } from "@/components/charts/rank-bar-chart";
+import { RankBarChart } from "@/components/charts/rank-bar-chart-lazy";
 import { Badge } from "@/components/ui/badge";
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
 import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";

package/components/overview/section-skeletons.tsx

@@ -0,0 +1,35 @@
+type SectionSkeletonProps = {
+  heightClass?: string;
+  label: string;
+};
+
+export function OverviewSectionSkeleton({ heightClass = "h-48", label }: SectionSkeletonProps) {
+  return (
+    <div
+      className={`flex items-center justify-center rounded-md border border-dashed border-muted-foreground/30 bg-muted/30 text-xs text-muted-foreground ${heightClass}`}
+      role="status"
+      aria-live="polite"
+    >
+      {label}
+    </div>
+  );
+}
+
+export function OverviewPrimarySkeleton() {
+  return (
+    <div className="space-y-6">
+      <OverviewSectionSkeleton heightClass="h-20" label="Loading usage pulse…" />
+      <OverviewSectionSkeleton heightClass="h-36" label="Loading summary cards…" />
+      <OverviewSectionSkeleton heightClass="h-72" label="Loading trend chart…" />
+    </div>
+  );
+}
+
+export function OverviewRepairSkeleton() {
+  return (
+    <div className="space-y-6">
+      <OverviewSectionSkeleton heightClass="h-24" label="Loading review status…" />
+      <OverviewSectionSkeleton heightClass="h-40" label="Loading repair items…" />
+    </div>
+  );
+}

package/dist/runtime/agent.mjs

@@ -655,6 +655,11 @@ var init_client = __esm({
     dbPath = process.env.TOKENTRACE_DB ?? databaseUrlPath(process.env.DATABASE_URL) ?? defaultDbPath;
     fs.mkdirSync(path.dirname(dbPath), { recursive: true });
     sqlite = new Database(dbPath);
+    sqlite.pragma("journal_mode = WAL");
+    sqlite.pragma("synchronous = NORMAL");
+    sqlite.pragma("temp_store = MEMORY");
+    sqlite.pragma("cache_size = -65536");
+    sqlite.pragma("mmap_size = 268435456");
     sqlite.pragma("busy_timeout = 10000");
     sqlite.pragma("foreign_keys = ON");
     registerSqliteFunctions(sqlite);
@@ -765,6 +770,24 @@ var init_agent_actions = __esm({
   }
 });
 
+// src/db/prepared.ts
+function prepareCached(sql2) {
+  let stmt = cache.get(sql2);
+  if (!stmt) {
+    stmt = sqlite.prepare(sql2);
+    cache.set(sql2, stmt);
+  }
+  return stmt;
+}
+var cache;
+var init_prepared = __esm({
+  "src/db/prepared.ts"() {
+    "use strict";
+    init_client();
+    cache = /* @__PURE__ */ new Map();
+  }
+});
+
 // src/lib/analytics-query-helpers.ts
 function number(value) {
   return typeof value === "number" && Number.isFinite(value) ? value : 0;
@@ -822,7 +845,7 @@ function parseJson(value, fallback) {
   }
 }
 function rows(sql2, ...params) {
-  return sqlite.prepare(sql2).all(...params);
+  return prepareCached(sql2).all(...params);
 }
 function withQuery(path3, params) {
   const query = new URLSearchParams();
@@ -877,7 +900,7 @@ function timestampJoinCondition(filters = {}, alias = "i") {
 var init_analytics_query_helpers = __esm({
   "src/lib/analytics-query-helpers.ts"() {
     "use strict";
-    init_client();
+    init_prepared();
   }
 });
 
@@ -3419,7 +3442,7 @@ var init_repair_actions = __esm({
 
 // src/lib/unknown-cost-repair/suggestions.ts
 function rows2(sql2, ...params) {
-  return sqlite.prepare(sql2).all(...params);
+  return prepareCached(sql2).all(...params);
 }
 function buildPricedModelLookup() {
   const pricedRows = rows2(
@@ -3493,7 +3516,7 @@ function aliasSuggestion({
 var init_suggestions = __esm({
   "src/lib/unknown-cost-repair/suggestions.ts"() {
     "use strict";
-    init_client();
+    init_prepared();
     init_model_aliases();
   }
 });
@@ -3503,7 +3526,7 @@ function number3(value) {
   return Number(value ?? 0);
 }
 function rows3(sql2, ...params) {
-  return sqlite.prepare(sql2).all(...params);
+  return prepareCached(sql2).all(...params);
 }
 function interactionDateFilter(filters = {}, alias = "i") {
   const clauses = [];
@@ -3691,7 +3714,7 @@ function buildUnknownCostRepairWorkbench(filters = {}, options = {}) {
 var init_workbench = __esm({
   "src/lib/unknown-cost-repair/workbench.ts"() {
     "use strict";
-    init_client();
+    init_prepared();
     init_repair_actions();
     init_keys();
     init_reviews();

package/dist/runtime/db-migrate.mjs

@@ -628,6 +628,11 @@ function databaseUrlPath(value) {
 var dbPath = process.env.TOKENTRACE_DB ?? databaseUrlPath(process.env.DATABASE_URL) ?? defaultDbPath;
 fs.mkdirSync(path.dirname(dbPath), { recursive: true });
 var sqlite = new Database(dbPath);
+sqlite.pragma("journal_mode = WAL");
+sqlite.pragma("synchronous = NORMAL");
+sqlite.pragma("temp_store = MEMORY");
+sqlite.pragma("cache_size = -65536");
+sqlite.pragma("mmap_size = 268435456");
 sqlite.pragma("busy_timeout = 10000");
 sqlite.pragma("foreign_keys = ON");
 registerSqliteFunctions(sqlite);

package/dist/runtime/db-seed.mjs

@@ -628,6 +628,11 @@ function databaseUrlPath(value) {
 var dbPath = process.env.TOKENTRACE_DB ?? databaseUrlPath(process.env.DATABASE_URL) ?? defaultDbPath;
 fs.mkdirSync(path.dirname(dbPath), { recursive: true });
 var sqlite = new Database(dbPath);
+sqlite.pragma("journal_mode = WAL");
+sqlite.pragma("synchronous = NORMAL");
+sqlite.pragma("temp_store = MEMORY");
+sqlite.pragma("cache_size = -65536");
+sqlite.pragma("mmap_size = 268435456");
 sqlite.pragma("busy_timeout = 10000");
 sqlite.pragma("foreign_keys = ON");
 registerSqliteFunctions(sqlite);

package/dist/runtime/digest.mjs

@@ -655,6 +655,11 @@ var init_client = __esm({
     dbPath = process.env.TOKENTRACE_DB ?? databaseUrlPath(process.env.DATABASE_URL) ?? defaultDbPath;
     fs.mkdirSync(path.dirname(dbPath), { recursive: true });
     sqlite = new Database(dbPath);
+    sqlite.pragma("journal_mode = WAL");
+    sqlite.pragma("synchronous = NORMAL");
+    sqlite.pragma("temp_store = MEMORY");
+    sqlite.pragma("cache_size = -65536");
+    sqlite.pragma("mmap_size = 268435456");
     sqlite.pragma("busy_timeout = 10000");
     sqlite.pragma("foreign_keys = ON");
     registerSqliteFunctions(sqlite);
@@ -663,6 +668,24 @@ var init_client = __esm({
   }
 });
 
+// src/db/prepared.ts
+function prepareCached(sql2) {
+  let stmt = cache.get(sql2);
+  if (!stmt) {
+    stmt = sqlite.prepare(sql2);
+    cache.set(sql2, stmt);
+  }
+  return stmt;
+}
+var cache;
+var init_prepared = __esm({
+  "src/db/prepared.ts"() {
+    "use strict";
+    init_client();
+    cache = /* @__PURE__ */ new Map();
+  }
+});
+
 // src/lib/analytics-query-helpers.ts
 function number(value) {
   return typeof value === "number" && Number.isFinite(value) ? value : 0;
@@ -720,7 +743,7 @@ function parseJson(value, fallback) {
   }
 }
 function rows(sql2, ...params) {
-  return sqlite.prepare(sql2).all(...params);
+  return prepareCached(sql2).all(...params);
 }
 function withQuery(path2, params) {
   const query = new URLSearchParams();
@@ -775,7 +798,7 @@ function timestampJoinCondition(filters = {}, alias = "i") {
 var init_analytics_query_helpers = __esm({
   "src/lib/analytics-query-helpers.ts"() {
     "use strict";
-    init_client();
+    init_prepared();
   }
 });