tokentrace 0.1.0

package/bin/tokentrace.js

@@ -0,0 +1,316 @@
+#!/usr/bin/env node
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { createInterface } from "node:readline/promises";
+import { stdin as input, stdout as output } from "node:process";
+import { fileURLToPath } from "node:url";
+import getPort, { portNumbers } from "get-port";
+import open from "open";
+
+const binPath = fs.realpathSync(fileURLToPath(import.meta.url));
+const packageRoot = path.resolve(path.dirname(binPath), "..");
+const invocationCwd = process.cwd();
+const packageJson = JSON.parse(
+  fs.readFileSync(path.join(packageRoot, "package.json"), "utf8")
+);
+
+function help() {
+  return `TokenTrace CLI
+
+Usage:
+  tokentrace              Start local dashboard
+  tokentrace serve        Start local dashboard
+  tokentrace scan         Scan local AI CLI usage logs
+  tokentrace run <cmd>    Run a command and record wrapper diagnostics
+  tokentrace reset        Reset local database
+  tokentrace --version    Print version`;
+}
+
+function appDataDir() {
+  if (process.env.TOKENTRACE_HOME) return path.resolve(process.env.TOKENTRACE_HOME);
+  const home = os.homedir();
+  if (process.platform === "darwin") {
+    return path.join(home, "Library", "Application Support", "TokenTrace");
+  }
+  if (process.platform === "win32") {
+    return path.join(process.env.APPDATA ?? path.join(home, "AppData", "Roaming"), "TokenTrace");
+  }
+  return path.join(process.env.XDG_DATA_HOME ?? path.join(home, ".local", "share"), "tokentrace");
+}
+
+function runtimeEnv() {
+  const dataDir = appDataDir();
+  fs.mkdirSync(dataDir, { recursive: true });
+  const dbPath = path.join(dataDir, "tokentrace.db");
+  return {
+    ...process.env,
+    TOKENTRACE_DB: process.env.TOKENTRACE_DB ?? dbPath,
+    DATABASE_URL: process.env.DATABASE_URL ?? `file:${dbPath}`,
+    TOKENTRACE_APP_DATA_DIR: dataDir,
+    TOKENTRACE_WORKDIR: invocationCwd,
+    NEXT_TELEMETRY_DISABLED: "1"
+  };
+}
+
+function nextBin() {
+  return path.join(packageRoot, "node_modules", "next", "dist", "bin", "next");
+}
+
+function runtimeScriptPath(scriptName) {
+  const compiled = path.join(packageRoot, "dist", "runtime", `${scriptName}.mjs`);
+  if (fs.existsSync(compiled)) return compiled;
+  return path.join(packageRoot, "scripts", `${scriptName}.ts`);
+}
+
+function scriptCommand(scriptName, args) {
+  const scriptPath = runtimeScriptPath(scriptName);
+  if (scriptPath.endsWith(".mjs")) {
+    return [process.execPath, [scriptPath, ...args]];
+  }
+  const tsx = path.join(packageRoot, "node_modules", "tsx", "dist", "cli.mjs");
+  return [process.execPath, [tsx, scriptPath, ...args]];
+}
+
+function runNodeScript(scriptName, args = [], options = {}) {
+  return new Promise((resolve, reject) => {
+    const [command, commandArgs] = scriptCommand(scriptName, args);
+    const child = spawn(command, commandArgs, {
+      cwd: packageRoot,
+      env: runtimeEnv(),
+      stdio: options.stdio ?? "inherit"
+    });
+    child.on("error", reject);
+    child.on("exit", (code) => {
+      if (code === 0) resolve();
+      else reject(new Error(`${scriptName} exited with code ${code}`));
+    });
+  });
+}
+
+async function initializeDatabase({ quiet = false } = {}) {
+  const env = runtimeEnv();
+  if (!quiet) {
+    console.log(`TokenTrace data: ${env.TOKENTRACE_APP_DATA_DIR}`);
+  }
+  await runNodeScript("db-migrate", [], { stdio: quiet ? "ignore" : "inherit" });
+  await runNodeScript("db-seed", [], { stdio: quiet ? "ignore" : "inherit" });
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function waitForServer(url, child) {
+  const deadline = Date.now() + 30_000;
+  while (Date.now() < deadline) {
+    if (child.exitCode != null) {
+      throw new Error(`TokenTrace server exited with code ${child.exitCode}`);
+    }
+    try {
+      const response = await fetch(url, { method: "HEAD" });
+      if (response.ok) return;
+    } catch {
+      // Keep polling until the server is ready or the timeout expires.
+    }
+    await sleep(300);
+  }
+  throw new Error("Timed out waiting for the TokenTrace server to start.");
+}
+
+async function serve() {
+  const buildId = path.join(packageRoot, ".next", "BUILD_ID");
+  if (!fs.existsSync(buildId)) {
+    console.error("TokenTrace is not built yet. Run `npm run build` before using the package CLI from a source checkout.");
+    process.exit(1);
+  }
+
+  await initializeDatabase();
+  const port = await getPort({ port: portNumbers(3030, 3999) });
+  const hostname = "127.0.0.1";
+  const url = `http://localhost:${port}`;
+
+  console.log(`Starting TokenTrace at ${url}`);
+  console.log("Press Ctrl+C to stop the server.");
+
+  const child = spawn(
+    process.execPath,
+    [nextBin(), "start", "--hostname", hostname, "--port", String(port)],
+    {
+      cwd: packageRoot,
+      env: {
+        ...runtimeEnv(),
+        PORT: String(port),
+        HOSTNAME: hostname
+      },
+      stdio: "inherit"
+    }
+  );
+
+  const stop = () => {
+    if (!child.killed) child.kill("SIGINT");
+  };
+  process.on("SIGINT", stop);
+  process.on("SIGTERM", stop);
+
+  try {
+    await waitForServer(url, child);
+    await open(url).catch(() => {
+      console.log(`Open this URL in your browser: ${url}`);
+    });
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : "Failed to start TokenTrace.");
+  }
+
+  child.on("exit", (code) => process.exit(code ?? 0));
+}
+
+async function scan(args) {
+  await initializeDatabase({ quiet: true });
+  await runNodeScript("scan", args);
+}
+
+async function reset(args) {
+  await initializeDatabase({ quiet: true });
+  if (!args.includes("--yes")) {
+    const rl = createInterface({ input, output });
+    const answer = await rl.question(
+      "Reset TokenTrace imported data and scan history? Settings and pricing will be kept. Continue? [y/N] "
+    );
+    rl.close();
+    if (!/^y(es)?$/i.test(answer.trim())) {
+      console.log("Reset cancelled.");
+      return;
+    }
+  }
+  await runNodeScript("reset", []);
+}
+
+function looksStructured(text) {
+  const trimmed = text.trim();
+  if (!trimmed || trimmed.length > 100_000) return null;
+  try {
+    const parsed = JSON.parse(trimmed);
+    if (parsed && typeof parsed === "object") return parsed;
+  } catch {
+    // Not structured JSON.
+  }
+  return null;
+}
+
+async function runWrapped(args) {
+  if (!args.length) {
+    console.error("Usage: tokentrace run <command> [args...]");
+    process.exit(1);
+  }
+
+  const env = runtimeEnv();
+  fs.mkdirSync(path.join(env.TOKENTRACE_APP_DATA_DIR, "wrapper-runs"), {
+    recursive: true
+  });
+
+  const [command, ...commandArgs] = args;
+  const startedAt = new Date();
+  let stdoutBytes = 0;
+  let stderrBytes = 0;
+  const stdoutChunks = [];
+  const stderrChunks = [];
+
+  const child = spawn(command, commandArgs, {
+    cwd: invocationCwd,
+    env: process.env,
+    stdio: ["inherit", "pipe", "pipe"],
+    shell: process.platform === "win32"
+  });
+
+  child.stdout.on("data", (chunk) => {
+    stdoutBytes += chunk.length;
+    if (stdoutBytes <= 100_000) stdoutChunks.push(chunk);
+    process.stdout.write(chunk);
+  });
+  child.stderr.on("data", (chunk) => {
+    stderrBytes += chunk.length;
+    if (stderrBytes <= 100_000) stderrChunks.push(chunk);
+    process.stderr.write(chunk);
+  });
+
+  const exitCode = await new Promise((resolve, reject) => {
+    child.on("error", reject);
+    child.on("exit", (code) => resolve(code ?? 0));
+  });
+
+  const endedAt = new Date();
+  const durationMs = endedAt.getTime() - startedAt.getTime();
+  const stdoutSample = Buffer.concat(stdoutChunks).toString("utf8");
+  const stderrSample = Buffer.concat(stderrChunks).toString("utf8");
+  const structuredOutput = looksStructured(stdoutSample);
+  const sessionId = `wrapper-${randomUUID()}`;
+  const record = {
+    timestamp: endedAt.toISOString(),
+    session_id: sessionId,
+    role: "tool",
+    type: "tokentrace.wrapper_run",
+    cwd: invocationCwd,
+    content: `Wrapper run for ${command} completed in ${durationMs}ms with ${stdoutBytes} stdout bytes and ${stderrBytes} stderr bytes.`,
+    command,
+    args: commandArgs,
+    duration_ms: durationMs,
+    stdout_bytes: stdoutBytes,
+    stderr_bytes: stderrBytes,
+    exit_code: exitCode,
+    structured_output_detected: Boolean(structuredOutput),
+    structured_output_preview: structuredOutput ?? undefined
+  };
+  const logPath = path.join(env.TOKENTRACE_APP_DATA_DIR, "wrapper-runs", "runs.jsonl");
+  fs.appendFileSync(logPath, `${JSON.stringify(record)}\n`);
+
+  console.log("");
+  console.log("TokenTrace wrapper summary");
+  console.log(`Duration: ${durationMs}ms`);
+  console.log(`stdout bytes: ${stdoutBytes}`);
+  console.log(`stderr bytes: ${stderrBytes}`);
+  console.log(`exit code: ${exitCode}`);
+  console.log(`diagnostic log: ${logPath}`);
+
+  process.exit(exitCode);
+}
+
+async function main() {
+  const [command = "serve", ...args] = process.argv.slice(2);
+
+  if (command === "--help" || command === "-h" || command === "help") {
+    console.log(help());
+    return;
+  }
+  if (command === "--version" || command === "-v") {
+    console.log(packageJson.version);
+    return;
+  }
+  if (command === "serve") {
+    await serve();
+    return;
+  }
+  if (command === "scan") {
+    await scan(args);
+    return;
+  }
+  if (command === "run") {
+    await runWrapped(args);
+    return;
+  }
+  if (command === "reset") {
+    await reset(args);
+    return;
+  }
+
+  console.error(`Unknown command: ${command}\n`);
+  console.error(help());
+  process.exit(1);
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : error);
+  process.exit(1);
+});

package/components/charts/rank-bar-chart.tsx

@@ -0,0 +1,69 @@
+"use client";
+
+import {
+  Bar,
+  BarChart,
+  CartesianGrid,
+  ResponsiveContainer,
+  Tooltip,
+  XAxis,
+  YAxis
+} from "recharts";
+import { formatCurrency, formatTokens } from "@/src/lib/format";
+
+export function RankBarChart({
+  data,
+  nameKey,
+  valueKey,
+  mode = "tokens",
+  color = "#ea580c"
+}: {
+  data: Array<Record<string, string | number | null>>;
+  nameKey: string;
+  valueKey: string;
+  mode?: "tokens" | "cost" | "count";
+  color?: string;
+}) {
+  const chartData = data
+    .slice(0, 8)
+    .map((item) => ({
+      name: String(item[nameKey] ?? "Unknown"),
+      value: Number(item[valueKey] ?? 0)
+    }))
+    .filter((item) => item.value > 0);
+
+  return (
+    <div className="h-72">
+      <ResponsiveContainer width="100%" height="100%">
+        <BarChart data={chartData} layout="vertical" margin={{ left: 16, right: 12, top: 8, bottom: 0 }}>
+          <CartesianGrid strokeDasharray="3 3" stroke="#e7ded3" />
+          <XAxis
+            type="number"
+            tick={{ fontSize: 12 }}
+            stroke="#786f65"
+            tickFormatter={(value) =>
+              mode === "cost" ? `$${Number(value).toFixed(0)}` : formatTokens(Number(value))
+            }
+          />
+          <YAxis
+            dataKey="name"
+            type="category"
+            width={120}
+            tick={{ fontSize: 12 }}
+            stroke="#786f65"
+          />
+          <Tooltip
+            formatter={(value) =>
+              mode === "cost"
+                ? formatCurrency(Number(value))
+                : mode === "tokens"
+                  ? `${formatTokens(Number(value))} tokens`
+                  : Number(value).toLocaleString()
+            }
+          />
+          <Bar dataKey="value" fill={color} radius={[0, 4, 4, 0]} />
+        </BarChart>
+      </ResponsiveContainer>
+    </div>
+  );
+}

package/components/charts/trend-chart.tsx

@@ -0,0 +1,123 @@
+"use client";
+
+import { useMemo, useState } from "react";
+import {
+  Area,
+  AreaChart,
+  CartesianGrid,
+  ResponsiveContainer,
+  Tooltip,
+  XAxis,
+  YAxis
+} from "recharts";
+import type { TrendPoint } from "@/src/lib/analytics";
+import { formatCurrency, formatShortDate, formatTokens } from "@/src/lib/format";
+import { Button } from "@/components/ui/button";
+
+type Period = "daily" | "weekly" | "monthly";
+
+function bucketFor(date: string, period: Period) {
+  const parsed = new Date(`${date}T00:00:00`);
+  if (period === "monthly") return `${parsed.getFullYear()}-${String(parsed.getMonth() + 1).padStart(2, "0")}-01`;
+  if (period === "weekly") {
+    const day = parsed.getDay() || 7;
+    parsed.setDate(parsed.getDate() - day + 1);
+    return parsed.toISOString().slice(0, 10);
+  }
+  return date;
+}
+
+export function TrendChart({
+  data,
+  metric,
+  color = "#0f766e"
+}: {
+  data: TrendPoint[];
+  metric: "totalTokens" | "cost";
+  color?: string;
+}) {
+  const [period, setPeriod] = useState<Period>("daily");
+  const chartData = useMemo(() => {
+    const buckets = new Map<string, TrendPoint>();
+    data.forEach((point) => {
+      const key = bucketFor(point.date, period);
+      const existing =
+        buckets.get(key) ??
+        ({
+          date: key,
+          totalTokens: 0,
+          inputTokens: 0,
+          outputTokens: 0,
+          cachedTokens: 0,
+          reasoningTokens: 0,
+          cost: 0
+        } satisfies TrendPoint);
+      existing.totalTokens += point.totalTokens;
+      existing.inputTokens += point.inputTokens;
+      existing.outputTokens += point.outputTokens;
+      existing.cachedTokens += point.cachedTokens;
+      existing.reasoningTokens += point.reasoningTokens;
+      existing.cost += point.cost;
+      buckets.set(key, existing);
+    });
+    return Array.from(buckets.values()).sort((a, b) => a.date.localeCompare(b.date));
+  }, [data, period]);
+
+  return (
+    <div className="space-y-3">
+      <div className="flex flex-wrap gap-2">
+        {(["daily", "weekly", "monthly"] as const).map((item) => (
+          <Button
+            key={item}
+            size="sm"
+            variant={period === item ? "default" : "outline"}
+            onClick={() => setPeriod(item)}
+          >
+            {item[0].toUpperCase() + item.slice(1)}
+          </Button>
+        ))}
+      </div>
+      <div className="h-72">
+        <ResponsiveContainer width="100%" height="100%">
+          <AreaChart data={chartData} margin={{ left: 0, right: 8, top: 8, bottom: 0 }}>
+            <defs>
+              <linearGradient id={`trend-${metric}`} x1="0" y1="0" x2="0" y2="1">
+                <stop offset="5%" stopColor={color} stopOpacity={0.35} />
+                <stop offset="95%" stopColor={color} stopOpacity={0.02} />
+              </linearGradient>
+            </defs>
+            <CartesianGrid strokeDasharray="3 3" stroke="#e7ded3" />
+            <XAxis
+              dataKey="date"
+              tickFormatter={formatShortDate}
+              tick={{ fontSize: 12 }}
+              stroke="#786f65"
+            />
+            <YAxis
+              tick={{ fontSize: 12 }}
+              stroke="#786f65"
+              tickFormatter={(value) =>
+                metric === "cost" ? `$${Number(value).toFixed(0)}` : formatTokens(Number(value))
+              }
+            />
+            <Tooltip
+              formatter={(value) =>
+                metric === "cost"
+                  ? formatCurrency(Number(value))
+                  : `${formatTokens(Number(value))} tokens`
+              }
+              labelFormatter={formatShortDate}
+            />
+            <Area
+              type="monotone"
+              dataKey={metric}
+              stroke={color}
+              fill={`url(#trend-${metric})`}
+              strokeWidth={2}
+            />
+          </AreaChart>
+        </ResponsiveContainer>
+      </div>
+    </div>
+  );
+}

package/components/empty-state.tsx

@@ -0,0 +1,14 @@
+import { Database } from "lucide-react";
+import { Card, CardContent } from "@/components/ui/card";
+
+export function EmptyState({ title, description }: { title: string; description: string }) {
+  return (
+    <Card>
+      <CardContent className="flex min-h-48 flex-col items-center justify-center gap-2 text-center">
+        <Database className="h-8 w-8 text-muted-foreground" />
+        <div className="font-medium">{title}</div>
+        <p className="max-w-md text-sm text-muted-foreground">{description}</p>
+      </CardContent>
+    </Card>
+  );
+}

package/components/pricing-settings.tsx

@@ -0,0 +1,171 @@
+"use client";
+
+import { useState, useTransition } from "react";
+import { Plus, Save } from "lucide-react";
+import type { PricingRow } from "@/src/lib/pricing";
+import { Button } from "@/components/ui/button";
+import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
+
+type EditablePricingRow = PricingRow & {
+  providerName?: string;
+};
+
+function numberInputValue(value: number | null) {
+  return value == null ? "" : String(value);
+}
+
+export function PricingSettings({ initialRows }: { initialRows: PricingRow[] }) {
+  const [rows, setRows] = useState<EditablePricingRow[]>(initialRows);
+  const [isPending, startTransition] = useTransition();
+  const [message, setMessage] = useState("");
+
+  function updateRow(index: number, patch: Partial<EditablePricingRow>) {
+    setRows((current) =>
+      current.map((row, rowIndex) => (rowIndex === index ? { ...row, ...patch } : row))
+    );
+  }
+
+  function addRow() {
+    setRows((current) => [
+      {
+        id: `new-${Date.now()}`,
+        providerId: "custom",
+        provider: "Custom",
+        providerName: "Custom",
+        model: "",
+        inputTokenPrice: null,
+        outputTokenPrice: null,
+        cachedInputTokenPrice: null,
+        currency: "USD",
+        effectiveFrom: null
+      },
+      ...current
+    ]);
+  }
+
+  function saveRow(row: EditablePricingRow) {
+    startTransition(async () => {
+      setMessage("");
+      const response = await fetch("/api/prices", {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({
+          providerId: row.providerId,
+          providerName: row.providerName ?? row.provider,
+          model: row.model,
+          inputTokenPrice: row.inputTokenPrice,
+          outputTokenPrice: row.outputTokenPrice,
+          cachedInputTokenPrice: row.cachedInputTokenPrice,
+          currency: row.currency
+        })
+      });
+      if (!response.ok) {
+        setMessage("Price save failed.");
+        return;
+      }
+      const latest = (await fetch("/api/prices").then((res) => res.json())) as PricingRow[];
+      setRows(latest);
+      setMessage("Price saved.");
+    });
+  }
+
+  return (
+    <div className="space-y-4">
+      <Card>
+        <CardHeader className="flex flex-col gap-3 sm:flex-row sm:items-center sm:justify-between">
+          <div>
+            <CardTitle>Model Pricing</CardTitle>
+            <CardDescription>
+              Prices are per 1M tokens. Seed values are editable placeholders.
+            </CardDescription>
+          </div>
+          <Button variant="outline" onClick={addRow}>
+            <Plus className="h-4 w-4" />
+            Add model
+          </Button>
+        </CardHeader>
+        <CardContent className="table-scroll">
+          <Table>
+            <TableHeader>
+              <TableRow>
+                <TableHead>Provider ID</TableHead>
+                <TableHead>Provider</TableHead>
+                <TableHead>Model</TableHead>
+                <TableHead>Input / 1M</TableHead>
+                <TableHead>Output / 1M</TableHead>
+                <TableHead>Cached input / 1M</TableHead>
+                <TableHead>Currency</TableHead>
+                <TableHead></TableHead>
+              </TableRow>
+            </TableHeader>
+            <TableBody>
+              {rows.map((row, index) => (
+                <TableRow key={row.id}>
+                  <TableCell>
+                    <Input value={row.providerId} onChange={(event) => updateRow(index, { providerId: event.target.value })} />
+                  </TableCell>
+                  <TableCell>
+                    <Input value={row.providerName ?? row.provider} onChange={(event) => updateRow(index, { providerName: event.target.value })} />
+                  </TableCell>
+                  <TableCell>
+                    <Input value={row.model} onChange={(event) => updateRow(index, { model: event.target.value })} />
+                  </TableCell>
+                  <TableCell>
+                    <Input
+                      inputMode="decimal"
+                      value={numberInputValue(row.inputTokenPrice)}
+                      onChange={(event) => updateRow(index, { inputTokenPrice: event.target.value === "" ? null : Number(event.target.value) })}
+                    />
+                  </TableCell>
+                  <TableCell>
+                    <Input
+                      inputMode="decimal"
+                      value={numberInputValue(row.outputTokenPrice)}
+                      onChange={(event) => updateRow(index, { outputTokenPrice: event.target.value === "" ? null : Number(event.target.value) })}
+                    />
+                  </TableCell>
+                  <TableCell>
+                    <Input
+                      inputMode="decimal"
+                      value={numberInputValue(row.cachedInputTokenPrice)}
+                      onChange={(event) => updateRow(index, { cachedInputTokenPrice: event.target.value === "" ? null : Number(event.target.value) })}
+                    />
+                  </TableCell>
+                  <TableCell>
+                    <Input value={row.currency} onChange={(event) => updateRow(index, { currency: event.target.value })} />
+                  </TableCell>
+                  <TableCell>
+                    <Button size="sm" onClick={() => saveRow(row)} disabled={isPending || !row.model}>
+                      <Save className="h-4 w-4" />
+                      Save
+                    </Button>
+                  </TableCell>
+                </TableRow>
+              ))}
+            </TableBody>
+          </Table>
+          {message ? <div className="mt-3 text-sm text-muted-foreground">{message}</div> : null}
+        </CardContent>
+      </Card>
+
+      <Card>
+        <CardHeader>
+          <CardTitle>Cost Formula</CardTitle>
+          <CardDescription>Costs are computed per interaction and then aggregated.</CardDescription>
+        </CardHeader>
+        <CardContent className="space-y-3 text-sm text-muted-foreground">
+          <Label>Formula</Label>
+          <div className="rounded-md border bg-muted/40 p-3 font-mono text-xs text-foreground">
+            input * inputPrice + output * outputPrice + cacheRead * cachedInputPrice + cacheWrite * inputPrice
+          </div>
+          <p>
+            TokenTrace separates exact token counts from estimated counts. Unknown model prices produce unknown costs.
+          </p>
+        </CardContent>
+      </Card>
+    </div>
+  );
+}