tokentrace 0.15.2 → 0.16.0

package/CHANGELOG.md

@@ -4,6 +4,49 @@ All notable changes to TokenTrace are documented here.
 
 ## Unreleased
 
+## [0.16.0] - 2026-05-23
+
+### Added
+
+- **Parser overrides.** Force a specific parser for a file or exclude it from
+  future scans without editing source code. Override is honored by the next
+  scan via `selectAdapter`. New surfaces:
+  - Dashboard: `Parser overrides` panel on `/parser-debug` (list, add, clear).
+  - REST: `GET / POST / DELETE /api/parser-overrides`.
+  - REST: `POST /api/parser-debug/preview` returns predicted parse output for
+    an alternate parser without writing to the local database.
+  - CLI: `tokentrace repair set-parser <path> --parser <id> [--note "..."]`,
+    `--exclude` variant, and `clear-parser <path>` (all support `--json`).
+  - File paths are normalized to absolute form so scan-resolved paths match
+    user-supplied overrides, regardless of relative vs absolute input.
+- **Saved reports.** Persisted local report templates that can be replayed
+  from the dashboard or the CLI. New surfaces:
+  - Dashboard: `/reports` page with create / list / delete.
+  - REST: `GET / POST /api/saved-reports`, `GET / DELETE /api/saved-reports/[id]`.
+  - CLI: `tokentrace report --saved "<name>" --format json|markdown|html`
+    and `tokentrace report --list-saved [--json]`.
+  - Standalone HTML report format escapes every user-supplied filter value
+    so opening or archiving the file is safe.
+  - Param keys are validated against an allow-list before persistence.
+- **Agent handoff.** Structured envelope plus action log for multi-agent
+  workflows.
+  - CLI: `tokentrace agent --handoff [--json]` returns the
+    `tokentrace.handoff.v1` envelope (scan state, repair queue, confidence,
+    recent actions, suggested next actions). Pure read.
+  - CLI: `tokentrace agent --actions [--limit N] [--json]` reads the local
+    agent action log.
+  - MCP: new `get_handoff` tool returns the same envelope; added to the
+    recommended workflow in `get_agent_guide`.
+  - Action log is bounded to the last 500 entries (configurable via
+    `TOKENTRACE_AGENT_ACTION_LOG_MAX`), token-shaped strings are redacted
+    before write, and the writer is best-effort so logging never breaks the
+    CLI command it observes. `tokentrace scan` now writes a row on completion.
+
+### Internal
+
+- Three new local SQLite tables: `file_parser_overrides`, `saved_reports`,
+  `agent_actions`. All additive migrations.
+
 ## [0.15.2] - 2026-05-23
 
 ### Fixed

package/app/api/parser-debug/preview/route.ts

@@ -0,0 +1,92 @@
+import fs from "node:fs/promises";
+import { NextResponse } from "next/server";
+import { adapters } from "@/src/ingestion/adapters";
+import type { FileCandidate, NormalizedInteraction } from "@/src/ingestion/types";
+
+export const dynamic = "force-dynamic";
+
+type PreviewBody = {
+  path?: unknown;
+  parserId?: unknown;
+};
+
+export async function POST(request: Request) {
+  let body: PreviewBody;
+  try {
+    body = (await request.json()) as PreviewBody;
+  } catch {
+    return NextResponse.json({ error: "Request body must be JSON." }, { status: 400 });
+  }
+
+  const filePath = typeof body.path === "string" ? body.path.trim() : "";
+  const parserId = typeof body.parserId === "string" ? body.parserId.trim() : "";
+
+  if (!filePath) {
+    return NextResponse.json({ error: "path is required" }, { status: 400 });
+  }
+  if (!parserId) {
+    return NextResponse.json({ error: "parserId is required" }, { status: 400 });
+  }
+
+  const adapter = adapters.find((candidate) => candidate.id === parserId);
+  if (!adapter) {
+    return NextResponse.json(
+      { error: `parser ${parserId} is not registered` },
+      { status: 404 }
+    );
+  }
+
+  let stat;
+  try {
+    stat = await fs.stat(filePath);
+  } catch {
+    return NextResponse.json({ error: `file not found: ${filePath}` }, { status: 404 });
+  }
+
+  const candidate: FileCandidate = {
+    path: filePath,
+    sizeBytes: stat.size,
+    modifiedTime: stat.mtime
+  };
+
+  let parseResult;
+  try {
+    parseResult = await adapter.parse(candidate, { storeRawMessageContent: false });
+  } catch (error) {
+    return NextResponse.json(
+      {
+        error: `parser ${parserId} threw while parsing: ${
+          error instanceof Error ? error.message : "Unknown error"
+        }`
+      },
+      { status: 422 }
+    );
+  }
+
+  const allInteractions: NormalizedInteraction[] = parseResult.sessions.flatMap(
+    (session) => session.interactions
+  );
+  const predictedTotalTokens = allInteractions.reduce(
+    (sum, interaction) => sum + (interaction.totalTokens ?? 0),
+    0
+  );
+
+  return NextResponse.json({
+    parserId,
+    path: filePath,
+    sessions: parseResult.sessions.map((session) => ({
+      externalId: session.externalId,
+      provider: session.provider,
+      tool: session.tool,
+      interactionCount: session.interactions.length,
+      totalTokens: session.interactions.reduce(
+        (sum, interaction) => sum + (interaction.totalTokens ?? 0),
+        0
+      )
+    })),
+    predictedInteractions: allInteractions.length,
+    predictedTotalTokens,
+    warnings: parseResult.warnings,
+    errors: parseResult.errors
+  });
+}

package/app/api/parser-overrides/route.ts

@@ -0,0 +1,65 @@
+import { NextResponse } from "next/server";
+import {
+  clearParserOverride,
+  listParserOverrides,
+  setParserOverride
+} from "@/src/lib/parser-overrides";
+
+export const dynamic = "force-dynamic";
+
+type WriteBody = {
+  path?: unknown;
+  parserId?: unknown;
+  excluded?: unknown;
+  note?: unknown;
+};
+
+export async function GET() {
+  return NextResponse.json({ overrides: listParserOverrides() });
+}
+
+export async function POST(request: Request) {
+  let body: WriteBody;
+  try {
+    body = (await request.json()) as WriteBody;
+  } catch {
+    return NextResponse.json({ error: "Request body must be JSON." }, { status: 400 });
+  }
+
+  const path = typeof body.path === "string" ? body.path.trim() : "";
+  if (!path) return NextResponse.json({ error: "path is required" }, { status: 400 });
+
+  const excluded = body.excluded === true;
+  const parserId =
+    !excluded && typeof body.parserId === "string" && body.parserId.trim()
+      ? body.parserId.trim()
+      : undefined;
+  const note = typeof body.note === "string" && body.note.trim() ? body.note.trim() : undefined;
+
+  if (!excluded && !parserId) {
+    return NextResponse.json({ error: "parserId or excluded must be provided" }, { status: 400 });
+  }
+
+  try {
+    const override = setParserOverride({ path, parserId, excluded, note });
+    return NextResponse.json({ override });
+  } catch (error) {
+    return NextResponse.json(
+      { error: error instanceof Error ? error.message : "Failed to set parser override" },
+      { status: 400 }
+    );
+  }
+}
+
+export async function DELETE(request: Request) {
+  let body: { path?: unknown };
+  try {
+    body = (await request.json()) as { path?: unknown };
+  } catch {
+    return NextResponse.json({ error: "Request body must be JSON." }, { status: 400 });
+  }
+  const path = typeof body.path === "string" ? body.path.trim() : "";
+  if (!path) return NextResponse.json({ error: "path is required" }, { status: 400 });
+  const removed = clearParserOverride(path);
+  return NextResponse.json({ removed, path });
+}

package/app/api/saved-reports/[id]/route.ts

@@ -0,0 +1,20 @@
+import { NextResponse } from "next/server";
+import { deleteSavedReport, findSavedReportById } from "@/src/lib/saved-reports-store";
+
+export const dynamic = "force-dynamic";
+
+type RouteContext = { params: Promise<{ id: string }> };
+
+export async function GET(_request: Request, context: RouteContext) {
+  const { id } = await context.params;
+  const report = findSavedReportById(id);
+  if (!report) return NextResponse.json({ error: "saved report not found" }, { status: 404 });
+  return NextResponse.json({ report });
+}
+
+export async function DELETE(_request: Request, context: RouteContext) {
+  const { id } = await context.params;
+  const removed = deleteSavedReport(id);
+  if (!removed) return NextResponse.json({ error: "saved report not found" }, { status: 404 });
+  return NextResponse.json({ removed: true, id });
+}

package/app/api/saved-reports/route.ts

@@ -0,0 +1,34 @@
+import { NextResponse } from "next/server";
+import { createSavedReport, listSavedReports } from "@/src/lib/saved-reports-store";
+
+export const dynamic = "force-dynamic";
+
+export async function GET() {
+  return NextResponse.json({ reports: listSavedReports() });
+}
+
+export async function POST(request: Request) {
+  let body: Record<string, unknown>;
+  try {
+    body = (await request.json()) as Record<string, unknown>;
+  } catch {
+    return NextResponse.json({ error: "Request body must be JSON." }, { status: 400 });
+  }
+
+  const name = typeof body.name === "string" ? body.name : "";
+  const viewType = typeof body.viewType === "string" ? body.viewType : "";
+  const format = typeof body.format === "string" ? body.format : undefined;
+  const params =
+    body.params && typeof body.params === "object" && !Array.isArray(body.params)
+      ? (body.params as Record<string, string | number | boolean>)
+      : undefined;
+
+  try {
+    const report = createSavedReport({ name, viewType, format, params });
+    return NextResponse.json({ report }, { status: 201 });
+  } catch (error) {
+    const message = error instanceof Error ? error.message : "Failed to create saved report";
+    const status = /already exists/i.test(message) ? 409 : 400;
+    return NextResponse.json({ error: message }, { status });
+  }
+}

package/app/parser-debug/page.tsx

@@ -4,7 +4,10 @@ import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@
 import { MonoText, PageHeader } from "@/components/ui/typography";
 import { EmptyState } from "@/components/empty-state";
 import { ScanNowButton } from "@/components/scan-now-button";
+import { ParserOverridesPanel } from "@/components/parser-debug/parser-overrides-panel";
 import { getScanTrustData } from "@/src/lib/analytics";
+import { adapters } from "@/src/ingestion/adapters";
+import { listParserOverrides } from "@/src/lib/parser-overrides";
 
 export const dynamic = "force-dynamic";
 
@@ -16,6 +19,11 @@ export default async function ParserDebugPage({
   const params = await searchParams;
   const selectedSource = params?.source;
   const { scanFiles, health } = getScanTrustData();
+  const initialOverrides = listParserOverrides();
+  const parserOptions = adapters.map((adapter) => ({
+    id: adapter.id,
+    displayName: adapter.displayName
+  }));
   const visibleScanFiles = selectedSource
     ? scanFiles.filter((file) => file.path === selectedSource)
     : scanFiles.slice(0, 500);
@@ -30,6 +38,8 @@ export default async function ParserDebugPage({
         description="Inspect adapter selection, parser confidence, extracted tokens, warnings, and failures."
       />
 
+      <ParserOverridesPanel initialOverrides={initialOverrides} parsers={parserOptions} />
+
       <div className="rounded-md border bg-card p-3">
         <div className="text-sm font-medium">Latest parser mix</div>
         <div className="mt-2 flex flex-wrap gap-2">

package/app/reports/page.tsx

@@ -0,0 +1,19 @@
+import { PageHeader } from "@/components/ui/typography";
+import { SavedReportsPanel } from "@/components/reports/saved-reports-panel";
+import { listSavedReports } from "@/src/lib/saved-reports-store";
+
+export const dynamic = "force-dynamic";
+
+export default async function ReportsPage() {
+  const reports = listSavedReports();
+
+  return (
+    <div className="space-y-6">
+      <PageHeader
+        title="Reports"
+        description="Save reusable local report templates and replay them from the CLI."
+      />
+      <SavedReportsPanel initial={reports} />
+    </div>
+  );
+}

package/components/parser-debug/parser-overrides-panel.tsx

@@ -0,0 +1,205 @@
+"use client";
+
+import { useState, useTransition } from "react";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
+import { Badge } from "@/components/ui/badge";
+import { MonoText } from "@/components/ui/typography";
+import type { ParserOverride } from "@/src/lib/parser-overrides";
+
+type ParserOption = { id: string; displayName: string };
+
+type Props = {
+  initialOverrides: ParserOverride[];
+  parsers: ParserOption[];
+};
+
+export function ParserOverridesPanel({ initialOverrides, parsers }: Props) {
+  const [overrides, setOverrides] = useState<ParserOverride[]>(initialOverrides);
+  const [adding, setAdding] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const [busy, startTransition] = useTransition();
+
+  const [path, setPath] = useState("");
+  const [parserId, setParserId] = useState(parsers[0]?.id ?? "");
+  const [excluded, setExcluded] = useState(false);
+  const [note, setNote] = useState("");
+
+  function resetForm() {
+    setPath("");
+    setParserId(parsers[0]?.id ?? "");
+    setExcluded(false);
+    setNote("");
+    setError(null);
+  }
+
+  function submit() {
+    setError(null);
+    const trimmed = path.trim();
+    if (!trimmed) {
+      setError("Path is required.");
+      return;
+    }
+    const payload: {
+      path: string;
+      parserId?: string;
+      excluded?: boolean;
+      note?: string;
+    } = { path: trimmed };
+    if (excluded) payload.excluded = true;
+    else payload.parserId = parserId;
+    if (note.trim()) payload.note = note.trim();
+
+    startTransition(async () => {
+      const response = await fetch("/api/parser-overrides", {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify(payload)
+      });
+      const body = await response.json().catch(() => ({}));
+      if (!response.ok) {
+        setError(body.error ?? "Failed to set parser override.");
+        return;
+      }
+      setOverrides((prev) => {
+        const without = prev.filter((row) => row.path !== body.override.path);
+        return [body.override, ...without];
+      });
+      resetForm();
+      setAdding(false);
+    });
+  }
+
+  function clear(rowPath: string) {
+    setError(null);
+    startTransition(async () => {
+      const response = await fetch("/api/parser-overrides", {
+        method: "DELETE",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({ path: rowPath })
+      });
+      if (!response.ok) {
+        const body = await response.json().catch(() => ({}));
+        setError(body.error ?? "Failed to clear override.");
+        return;
+      }
+      setOverrides((prev) => prev.filter((row) => row.path !== rowPath));
+    });
+  }
+
+  return (
+    <Card>
+      <CardHeader className="flex flex-col gap-2 sm:flex-row sm:items-start sm:justify-between">
+        <div>
+          <CardTitle>Parser overrides</CardTitle>
+          <CardDescription>
+            Force a specific parser for a file or exclude it from future scans.
+          </CardDescription>
+        </div>
+        <Button
+          type="button"
+          size="sm"
+          variant={adding ? "outline" : "default"}
+          onClick={() => setAdding((value) => !value)}
+        >
+          {adding ? "Cancel" : "Add override"}
+        </Button>
+      </CardHeader>
+      <CardContent className="space-y-4">
+        {error ? (
+          <div className="rounded-md border border-destructive/40 bg-destructive/5 p-3 text-sm text-destructive">
+            {error}
+          </div>
+        ) : null}
+
+        {adding ? (
+          <div className="space-y-3 rounded-md border bg-muted/30 p-3">
+            <label className="block space-y-1 text-sm">
+              <span className="font-medium">File path</span>
+              <Input
+                value={path}
+                onChange={(event) => setPath(event.target.value)}
+                placeholder="/absolute/path/to/file.jsonl"
+                spellCheck={false}
+              />
+            </label>
+            <label className="flex items-center gap-2 text-sm">
+              <input
+                type="checkbox"
+                checked={excluded}
+                onChange={(event) => setExcluded(event.target.checked)}
+              />
+              <span>Exclude this file from scans</span>
+            </label>
+            {!excluded ? (
+              <label className="block space-y-1 text-sm">
+                <span className="font-medium">Parser</span>
+                <select
+                  value={parserId}
+                  onChange={(event) => setParserId(event.target.value)}
+                  className="h-9 w-full rounded-md border bg-card px-2 text-sm"
+                >
+                  {parsers.map((parser) => (
+                    <option key={parser.id} value={parser.id}>
+                      {parser.displayName} ({parser.id})
+                    </option>
+                  ))}
+                </select>
+              </label>
+            ) : null}
+            <label className="block space-y-1 text-sm">
+              <span className="font-medium">Note (optional)</span>
+              <Input
+                value={note}
+                onChange={(event) => setNote(event.target.value)}
+                placeholder="Why this override exists"
+              />
+            </label>
+            <div className="flex justify-end gap-2">
+              <Button type="button" variant="outline" size="sm" onClick={() => setAdding(false)} disabled={busy}>
+                Cancel
+              </Button>
+              <Button type="button" size="sm" onClick={submit} disabled={busy}>
+                {busy ? "Saving…" : "Save override"}
+              </Button>
+            </div>
+          </div>
+        ) : null}
+
+        {overrides.length === 0 ? (
+          <p className="text-sm text-muted-foreground">No parser overrides set.</p>
+        ) : (
+          <ul className="divide-y rounded-md border">
+            {overrides.map((override) => (
+              <li key={override.path} className="flex flex-col gap-2 p-3 sm:flex-row sm:items-start sm:justify-between">
+                <div className="min-w-0">
+                  <MonoText className="break-all text-sm">{override.path}</MonoText>
+                  <div className="mt-1 flex flex-wrap items-center gap-2 text-xs">
+                    {override.excluded ? (
+                      <Badge variant="warning">Excluded</Badge>
+                    ) : (
+                      <Badge variant="outline">{override.parserId}</Badge>
+                    )}
+                    {override.note ? (
+                      <span className="text-muted-foreground">{override.note}</span>
+                    ) : null}
+                  </div>
+                </div>
+                <Button
+                  type="button"
+                  size="sm"
+                  variant="outline"
+                  onClick={() => clear(override.path)}
+                  disabled={busy}
+                >
+                  Clear
+                </Button>
+              </li>
+            ))}
+          </ul>
+        )}
+      </CardContent>
+    </Card>
+  );
+}