tokentrace 0.17.0 → 0.18.1

package/CHANGELOG.md

@@ -4,6 +4,93 @@ All notable changes to TokenTrace are documented here.
 
 ## Unreleased
 
+## [0.18.1] - 2026-05-28
+
+### Fixed
+
+- **Scheduled scans now coalesce concurrent runs.** `runDueScheduledScan`
+  fires on every overview page load; two overlapping loads (a second tab, a
+  quick refresh, a prefetch) previously both passed the due-check and both
+  called `runScan`, producing duplicate scan runs. A module-scoped in-flight
+  promise guard now coalesces concurrent callers into a single scan within
+  the server process.
+- **`tokentrace repair auto-classify --apply` is now atomic.** The alias
+  writes and cost backfills for a batch run inside a single SQLite
+  transaction, and `backfillAlias` wraps its per-row updates in a transaction
+  too, so a failure partway can no longer leave the local database
+  half-applied with inaccurate reported counts.
+- **`tokentrace query` surfaces range errors cleanly.** Range validation
+  (e.g. `--from` after `--to`, or a preset combined with explicit dates) now
+  prints the validation message plus usage and exits non-zero, instead of
+  letting a raw stack trace escape.
+- **Anomaly z-scores survive JSON.** The flat-window (`mad === 0`) severe
+  branch reported `Infinity`, which `JSON.stringify` turned into `null` and
+  silently dropped the signal. It now reports a finite sentinel z-score.
+- **Stricter query CLI parsing.** Empty `--flag=` values are rejected with a
+  clear "Missing value" message rather than flowing through as empty strings.
+
+### Internal
+
+- **Test suite no longer flakes under parallel load.** CLI/MCP integration
+  tests spawn `bin/tokentrace.js`, which itself spawns `db-migrate` and
+  `db-seed`, fanning out to ~3x the core count in node processes. The Vitest
+  fork pool is now capped at roughly half the available cores with one
+  generous global timeout, removing the scattered per-test timeout overrides.
+- Removed the dead `tailwind.config.ts` left over from the Tailwind 4
+  (CSS-first) migration.
+
+## [0.18.0] - 2026-05-28
+
+### Local intelligence
+
+- **Anomaly detection (`tokentrace anomalies`).** A new modified-z-score (MAD)
+  detector scores the daily token and cost trend against a trailing window
+  (default 14 days) and surfaces deviations as `notable`, `high`, or `severe`.
+  Pure-stats; spends zero AI tokens. New CLI flags: `--window=N` (3..60),
+  `--metric=tokens|cost|all`, `--json`.
+- **Structured query (`tokentrace query`).** A new deterministic
+  parameterized aggregation that lets agents ask precise local questions
+  without TokenTrace performing any NL parsing. Group by
+  `model|project|tool|session|day`, aggregate
+  `cost|totalTokens|interactions`, with optional preset/from/to range,
+  exact-match `model|project|tool` filters, and `--top` clamped to 200.
+  Routes through `prepareCached` for the warm-statement path.
+- **Auto-classifier (`tokentrace repair auto-classify`).** Each unknown-cost
+  workbench group now carries a deterministic `classification` field
+  produced by three rules in confidence order: `exact-model` (0.95),
+  `family-fragment` (0.70, normalized via `modelNameCandidates`), and
+  `parser-source` (0.45, same source file as priced examples). The CLI
+  supports `--apply --min-confidence=N` (floor 0.85) which writes each
+  qualifying exact-model or family-fragment suggestion to a new
+  `model_aliases` table and backfills cost for the matching unknown-cost
+  interactions; `--dry-run` previews without writing. parser-source
+  matches are skipped from `--apply` because they have no
+  (provider, observed-model) pair to persist.
+- **Persistent model aliases.** A new `model_aliases` table maps
+  `(provider_id, observed_model)` to a priced model. The cost
+  recalculation pass now LEFT JOINs the alias table so aliased costs
+  survive every re-seed, with metadata noting the rule and confidence.
+- **Interactive Query page (`/query`).** A new dashboard route exposes the
+  structured-query surface with a server-rendered form (group-by, metric,
+  range preset / from / to, model/project/tool filters, top N). Results
+  render as a deterministic table. Same SQL path as the CLI and MCP tool.
+- **Anomaly drill-down + chart markers.** Each anomaly date in the
+  overview Anomalies panel is now a link to `/?range=custom&from=DATE&to=DATE`
+  so clicking filters the entire overview to that day. The Trend chart
+  also renders colored Recharts `ReferenceDot` markers on flagged days
+  (red severe, amber high, gray notable) when the bucket is daily.
+- **"Auto-classify" column in Repair Items.** The repair workbench table
+  now shows the suggested priced model, classification rule, and
+  confidence percentage alongside the existing legacy suggestion.
+- **New MCP tools.** `get_anomalies`, `query_usage`, and
+  `get_classifications` are wired into `src/lib/mcp/tools.ts` with full
+  input schemas and `src/lib/mcp-server.ts` handlers that translate
+  structured arguments into CLI flags. Each handler is read-only and
+  spends zero AI tokens.
+- **Agent discovery catalog.** `tokentrace agent --json` now exposes
+  `anomalies`, `query`, and `auto-classify` commands so MCP-capable
+  agents can discover the new local-intelligence surface.
+
 ## [0.17.0] - 2026-05-23
 
 ### Performance

package/TOKENTRACE_AGENT.md

@@ -92,6 +92,35 @@ curl http://127.0.0.1:3030/api/roadmap
    tokentrace evidence --json
    ```
 
+## Local Intelligence (zero AI tokens)
+
+TokenTrace ships deterministic, local-only analytical surfaces that spend
+no AI tokens. The agent on the calling side handles any natural-language
+parsing; TokenTrace executes pure SQL and statistics.
+
+Spot unusual local usage days via a modified-z-score (MAD) detector:
+
+```bash
+tokentrace anomalies --json [--window=N] [--metric=tokens|cost|all]
+```
+
+Run a structured aggregation by model, project, tool, session, or day:
+
+```bash
+tokentrace query --group-by model --metric cost --range 7d --json
+tokentrace query --group-by day --metric totalTokens --from 2026-05-01 --to 2026-05-15 --json
+```
+
+Get deterministic classification suggestions for the unknown-cost queue:
+
+```bash
+tokentrace repair auto-classify --json [--min-confidence=N]
+```
+
+The matching MCP tools are `get_anomalies`, `query_usage`, and
+`get_classifications`. All three are read-only and require no
+`confirmLocalScan` acknowledgement.
+
 ## Guardrails
 
 - Do not run `tokentrace reset` unless the human explicitly asks to clear imported local data.

package/app/page.tsx

@@ -2,6 +2,9 @@ import { Suspense } from "react";
 import Link from "next/link";
 import { ArrowRight } from "lucide-react";
 import { TrendSection } from "@/components/charts/trend-section-lazy";
+import type { TrendAnomalyMarker } from "@/components/charts/trend-chart";
+import { OverviewAnomaliesPanel } from "@/components/overview/anomalies-panel";
+import { detectAnomalies } from "@/src/lib/anomaly-detection";
 import { OverviewCurrentMixPanel } from "@/components/overview/current-mix-panel";
 import { DataConfidenceStrip } from "@/components/overview/data-confidence-strip";
 import { FirstRunPanel } from "@/components/overview/first-run-panel";
@@ -41,6 +44,13 @@ async function OverviewPrimarySection({ range }: { range: ResolvedDateRange }) {
     rangeLinkParams
   } = await getOverviewPrimaryData(range);
   const repairFocusHref = mergeHrefParams("/repair", rangeLinkParams);
+  const anomalyReport = detectAnomalies(data.trends);
+  const tokenMarkers: TrendAnomalyMarker[] = anomalyReport.anomalies
+    .filter((entry) => entry.metric === "tokens")
+    .map((entry) => ({ date: entry.date, value: entry.value, severity: entry.severity }));
+  const costMarkers: TrendAnomalyMarker[] = anomalyReport.anomalies
+    .filter((entry) => entry.metric === "cost")
+    .map((entry) => ({ date: entry.date, value: entry.value, severity: entry.severity }));
 
   return (
     <div className="space-y-8">
@@ -63,7 +73,13 @@ async function OverviewPrimarySection({ range }: { range: ResolvedDateRange }) {
         />
       </div>
       <OverviewTrustFooter health={trust.health} pricedModelCount={trust.pricedModelCount} />
-      <TrendSection data={data.trends} defaultWindow={trendDefaultWindow} />
+      <TrendSection
+        data={data.trends}
+        defaultWindow={trendDefaultWindow}
+        tokenMarkers={tokenMarkers}
+        costMarkers={costMarkers}
+      />
+      <OverviewAnomaliesPanel trends={data.trends} />
       <UsageGuardrailsPanel progress={data.usageGuardrails} />
       <OverviewRecommendationsCard recommendations={data.recommendations} />
       <OverviewCurrentMixPanel

package/app/query/page.tsx

@@ -0,0 +1,292 @@
+import { PageHeader } from "@/components/ui/typography";
+import { Badge } from "@/components/ui/badge";
+import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
+import {
+  Table,
+  TableBody,
+  TableCell,
+  TableHead,
+  TableHeader,
+  TableRow
+} from "@/components/ui/table";
+import {
+  runStructuredQuery,
+  type StructuredQueryArgs,
+  type StructuredQueryGroupBy,
+  type StructuredQueryMetric,
+  type StructuredQueryRangePreset,
+  type StructuredQuerySort
+} from "@/src/lib/structured-query";
+import { formatTokens } from "@/src/lib/format";
+
+export const dynamic = "force-dynamic";
+
+type QueryPageProps = {
+  searchParams?: Promise<Record<string, string | string[] | undefined>>;
+};
+
+function readString(value: string | string[] | undefined): string | undefined {
+  if (Array.isArray(value)) return value[0];
+  return typeof value === "string" && value.trim() ? value : undefined;
+}
+
+const GROUP_BY: StructuredQueryGroupBy[] = ["model", "project", "tool", "session", "day"];
+const METRIC: StructuredQueryMetric[] = ["cost", "totalTokens", "interactions"];
+const PRESETS: StructuredQueryRangePreset[] = ["today", "7d", "30d", "60d", "90d", "all"];
+const SORTS: StructuredQuerySort[] = ["desc", "asc"];
+
+function formatValue(value: number, metric: StructuredQueryMetric) {
+  if (metric === "cost") return `$${value.toFixed(2)}`;
+  if (metric === "totalTokens") return formatTokens(value);
+  return value.toLocaleString();
+}
+
+function selectClass() {
+  return "w-full rounded-md border bg-background px-3 py-2 text-sm shadow-sm focus:outline-none focus:ring-2 focus:ring-ring";
+}
+
+function inputClass() {
+  return "w-full rounded-md border bg-background px-3 py-2 text-sm shadow-sm focus:outline-none focus:ring-2 focus:ring-ring";
+}
+
+function buttonClass() {
+  return "inline-flex items-center justify-center rounded-md bg-primary px-4 py-2 text-sm font-medium text-primary-foreground shadow-sm hover:bg-primary/90";
+}
+
+export default async function QueryPage({ searchParams }: QueryPageProps) {
+  const params = (await searchParams) ?? {};
+  const groupBy = (readString(params.groupBy) as StructuredQueryGroupBy | undefined) ?? "model";
+  const metric = (readString(params.metric) as StructuredQueryMetric | undefined) ?? "cost";
+  const preset = readString(params.range) as StructuredQueryRangePreset | undefined;
+  const from = readString(params.from);
+  const to = readString(params.to);
+  const model = readString(params.model);
+  const project = readString(params.project);
+  const tool = readString(params.tool);
+  const topRaw = readString(params.topN);
+  const sort = (readString(params.sort) as StructuredQuerySort | undefined) ?? "desc";
+
+  const submitted = Boolean(params.submit);
+
+  let error: string | null = null;
+  let result: ReturnType<typeof runStructuredQuery> | null = null;
+
+  if (submitted) {
+    const args: StructuredQueryArgs = {
+      groupBy: GROUP_BY.includes(groupBy) ? groupBy : "model",
+      metric: METRIC.includes(metric) ? metric : "cost",
+      sort,
+      filters: { model, project, tool },
+      range: preset || from || to ? { preset, from, to } : undefined,
+      topN: topRaw ? Number(topRaw) : undefined
+    };
+    try {
+      result = runStructuredQuery(args);
+    } catch (caught) {
+      error = caught instanceof Error ? caught.message : "Query failed.";
+    }
+  }
+
+  return (
+    <div className="space-y-6">
+      <PageHeader
+        title="Query"
+        description="Deterministic local SQL aggregation. The form below runs entirely on your machine — no AI tokens are spent."
+      />
+
+      <Card>
+        <CardHeader>
+          <CardTitle>Run a structured query</CardTitle>
+          <CardDescription>
+            Group local interactions by model, project, tool, session, or day and aggregate cost,
+            total tokens, or interaction count.
+          </CardDescription>
+        </CardHeader>
+        <CardContent>
+          <form method="get" className="grid gap-4 md:grid-cols-3">
+            <label className="space-y-1 text-sm">
+              <span className="font-medium">Group by</span>
+              <select name="groupBy" defaultValue={groupBy} className={selectClass()}>
+                {GROUP_BY.map((value) => (
+                  <option key={value} value={value}>
+                    {value}
+                  </option>
+                ))}
+              </select>
+            </label>
+            <label className="space-y-1 text-sm">
+              <span className="font-medium">Metric</span>
+              <select name="metric" defaultValue={metric} className={selectClass()}>
+                {METRIC.map((value) => (
+                  <option key={value} value={value}>
+                    {value}
+                  </option>
+                ))}
+              </select>
+            </label>
+            <label className="space-y-1 text-sm">
+              <span className="font-medium">Sort</span>
+              <select name="sort" defaultValue={sort} className={selectClass()}>
+                {SORTS.map((value) => (
+                  <option key={value} value={value}>
+                    {value}
+                  </option>
+                ))}
+              </select>
+            </label>
+            <label className="space-y-1 text-sm">
+              <span className="font-medium">Preset range</span>
+              <select name="range" defaultValue={preset ?? ""} className={selectClass()}>
+                <option value="">(none)</option>
+                {PRESETS.map((value) => (
+                  <option key={value} value={value}>
+                    {value}
+                  </option>
+                ))}
+              </select>
+            </label>
+            <label className="space-y-1 text-sm">
+              <span className="font-medium">From (YYYY-MM-DD)</span>
+              <input
+                type="text"
+                name="from"
+                defaultValue={from ?? ""}
+                placeholder="2026-05-01"
+                className={inputClass()}
+              />
+            </label>
+            <label className="space-y-1 text-sm">
+              <span className="font-medium">To (YYYY-MM-DD, exclusive)</span>
+              <input
+                type="text"
+                name="to"
+                defaultValue={to ?? ""}
+                placeholder="2026-05-15"
+                className={inputClass()}
+              />
+            </label>
+            <label className="space-y-1 text-sm">
+              <span className="font-medium">Model filter</span>
+              <input type="text" name="model" defaultValue={model ?? ""} className={inputClass()} />
+            </label>
+            <label className="space-y-1 text-sm">
+              <span className="font-medium">Project filter</span>
+              <input
+                type="text"
+                name="project"
+                defaultValue={project ?? ""}
+                className={inputClass()}
+              />
+            </label>
+            <label className="space-y-1 text-sm">
+              <span className="font-medium">Tool filter</span>
+              <input type="text" name="tool" defaultValue={tool ?? ""} className={inputClass()} />
+            </label>
+            <label className="space-y-1 text-sm">
+              <span className="font-medium">Top N (1..200)</span>
+              <input
+                type="number"
+                name="topN"
+                min={1}
+                max={200}
+                defaultValue={topRaw ?? "20"}
+                className={inputClass()}
+              />
+            </label>
+            <div className="md:col-span-3">
+              <button type="submit" name="submit" value="1" className={buttonClass()}>
+                Run query
+              </button>
+            </div>
+          </form>
+        </CardContent>
+      </Card>
+
+      {error ? (
+        <Card>
+          <CardHeader>
+            <CardTitle>Query error</CardTitle>
+            <CardDescription>The structured-query validator rejected the arguments.</CardDescription>
+          </CardHeader>
+          <CardContent>
+            <p className="text-sm text-destructive">{error}</p>
+          </CardContent>
+        </Card>
+      ) : null}
+
+      {result ? (
+        <Card>
+          <CardHeader className="flex flex-col gap-3 pb-3 lg:flex-row lg:items-start lg:justify-between">
+            <div>
+              <CardTitle>
+                {result.rows.length === 0 ? "No rows" : `${result.rows.length} rows`}
+              </CardTitle>
+              <CardDescription>
+                Group by <code>{result.groupBy}</code>, metric <code>{result.metric}</code>, sort{" "}
+                <code>{result.sort}</code>. {result.range.preset ? `Range: ${result.range.preset}. ` : ""}
+                {result.range.from || result.range.to
+                  ? `Window: ${result.range.from ?? "—"} → ${result.range.to ?? "—"}. `
+                  : ""}
+                Showing {result.rows.length} of {result.totalGroups} groups
+                {result.truncated ? " (truncated by topN)" : ""}.
+              </CardDescription>
+            </div>
+            <div className="flex gap-2">
+              {result.filters.model ? <Badge variant="secondary">model={result.filters.model}</Badge> : null}
+              {result.filters.project ? <Badge variant="secondary">project={result.filters.project}</Badge> : null}
+              {result.filters.tool ? <Badge variant="secondary">tool={result.filters.tool}</Badge> : null}
+            </div>
+          </CardHeader>
+          <CardContent>
+            {result.rows.length === 0 ? (
+              <p className="text-sm text-muted-foreground">
+                No groups matched. Try a different range or remove filters.
+              </p>
+            ) : (
+              <div className="overflow-x-auto">
+                <Table>
+                  <TableHeader>
+                    <TableRow>
+                      <TableHead>{result.groupBy}</TableHead>
+                      <TableHead className="text-right">{result.metric}</TableHead>
+                      <TableHead className="text-right">interactions</TableHead>
+                      <TableHead className="text-right">totalTokens</TableHead>
+                      <TableHead className="text-right">cost</TableHead>
+                    </TableRow>
+                  </TableHeader>
+                  <TableBody>
+                    {result.rows.map((row) => (
+                      <TableRow key={row.group}>
+                        <TableCell className="font-medium">{row.group}</TableCell>
+                        <TableCell className="text-right">
+                          {formatValue(row.value, result.metric)}
+                        </TableCell>
+                        <TableCell className="text-right">{row.interactions.toLocaleString()}</TableCell>
+                        <TableCell className="text-right">{formatTokens(row.totalTokens)}</TableCell>
+                        <TableCell className="text-right">${row.cost.toFixed(2)}</TableCell>
+                      </TableRow>
+                    ))}
+                  </TableBody>
+                </Table>
+              </div>
+            )}
+          </CardContent>
+        </Card>
+      ) : null}
+
+      {!submitted && !error ? (
+        <Card>
+          <CardHeader>
+            <CardTitle>About this page</CardTitle>
+            <CardDescription>
+              The form submits a GET request with structured arguments. The same query is available
+              from the CLI as <code>tokentrace query --group-by ... --metric ... --json</code> and
+              through the MCP <code>query_usage</code> tool. All three paths execute the same
+              deterministic SQL — zero AI tokens are spent.
+            </CardDescription>
+          </CardHeader>
+        </Card>
+      ) : null}
+    </div>
+  );
+}

package/components/charts/trend-chart.tsx

@@ -2,7 +2,7 @@
 
 import * as React from "react";
 import { useMemo, useState } from "react";
-import { Area, AreaChart, CartesianGrid, Tooltip, XAxis, YAxis } from "recharts";
+import { Area, AreaChart, CartesianGrid, ReferenceDot, Tooltip, XAxis, YAxis } from "recharts";
 import type { TrendPoint } from "@/src/lib/analytics";
 import { formatCurrency, formatShortDate, formatTokens } from "@/src/lib/format";
 import { Button } from "@/components/ui/button";
@@ -118,6 +118,18 @@ function bucketFor(date: string, period: TrendBucket) {
   return date;
 }
 
+export type TrendAnomalyMarker = {
+  date: string;
+  value: number;
+  severity: "notable" | "high" | "severe";
+};
+
+function severityColor(severity: TrendAnomalyMarker["severity"]) {
+  if (severity === "severe") return "#dc2626";
+  if (severity === "high") return "#f59e0b";
+  return "#a3a3a3";
+}
+
 export function TrendChart({
   data,
   metric,
@@ -125,7 +137,8 @@ export function TrendChart({
   defaultWindow = "60d",
   period: controlledPeriod,
   trendWindow: controlledTrendWindow,
-  showControls = true
+  showControls = true,
+  markers
 }: {
   data: TrendPoint[];
   metric: "totalTokens" | "cost";
@@ -134,6 +147,7 @@ export function TrendChart({
   period?: TrendBucket;
   trendWindow?: TrendWindow;
   showControls?: boolean;
+  markers?: TrendAnomalyMarker[];
 }) {
   const [internalPeriod, setInternalPeriod] = useState<TrendBucket>("daily");
   const [internalTrendWindow, setInternalTrendWindow] = useState<TrendWindow>(defaultWindow);
@@ -228,6 +242,24 @@ export function TrendChart({
               fill={`url(#trend-${metric})`}
               strokeWidth={2}
             />
+            {period === "daily" && markers
+              ? markers
+                  .filter((marker) =>
+                    chartData.some((point) => point.date === marker.date)
+                  )
+                  .map((marker) => (
+                    <ReferenceDot
+                      key={`${marker.date}:${metric}`}
+                      x={marker.date}
+                      y={marker.value}
+                      r={5}
+                      fill={severityColor(marker.severity)}
+                      stroke="#ffffff"
+                      strokeWidth={2}
+                      ifOverflow="extendDomain"
+                    />
+                  ))
+              : null}
           </AreaChart>
         ) : (
           <ChartSkeleton />

package/components/charts/trend-section.tsx

@@ -5,7 +5,13 @@ import { useState } from "react";
 import type { TrendPoint } from "@/src/lib/analytics";
 import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
 import { Badge } from "@/components/ui/badge";
-import { TrendChart, TrendControls, type TrendBucket, type TrendWindow } from "@/components/charts/trend-chart";
+import {
+  TrendChart,
+  TrendControls,
+  type TrendAnomalyMarker,
+  type TrendBucket,
+  type TrendWindow
+} from "@/components/charts/trend-chart";
 
 function trendWindowLabel(window: TrendWindow) {
   if (window === "30d") return "Showing latest 30 days";
@@ -16,10 +22,14 @@ function trendWindowLabel(window: TrendWindow) {
 
 export function TrendSection({
   data,
-  defaultWindow
+  defaultWindow,
+  tokenMarkers,
+  costMarkers
 }: {
   data: TrendPoint[];
   defaultWindow: TrendWindow;
+  tokenMarkers?: TrendAnomalyMarker[];
+  costMarkers?: TrendAnomalyMarker[];
 }) {
   const [period, setPeriod] = useState<TrendBucket>("daily");
   const [trendWindow, setTrendWindow] = useState<TrendWindow>(defaultWindow);
@@ -65,6 +75,7 @@ export function TrendSection({
               period={period}
               trendWindow={trendWindow}
               showControls={false}
+              markers={tokenMarkers}
             />
           </CardContent>
         </Card>
@@ -81,6 +92,7 @@ export function TrendSection({
               period={period}
               trendWindow={trendWindow}
               showControls={false}
+              markers={costMarkers}
             />
           </CardContent>
         </Card>