tokvista 1.9.0 → 1.11.0

package/README.md

@@ -147,6 +147,22 @@ npx tokvista convert tokens.json --to supernova --output tokens-sn.json
 npx tokvista convert tokens.json --to w3c
 ```
 
+### Build All Formats
+
+```bash
+# Build everything in one command
+npx tokvista build tokens.json --output-dir ./dist
+
+# Creates:
+# - tokens.css
+# - tokens.scss
+# - tokens.js
+# - tailwind.config.js
+
+# Skip validation for faster builds
+npx tokvista build tokens.json --output-dir ./dist --skip-validation
+```
+
 ### Interactive Setup
 
 ```bash
@@ -179,6 +195,7 @@ Then run `npx tokvista` to use your config.
 | `tokvista validate <file>` | Validate token structure and values |
 | `tokvista diff <old> <new>` | Compare two token files |
 | `tokvista convert <file> --to <format>` | Convert between token formats |
+| `tokvista build <file> --output-dir <dir>` | Build all formats (validate + export) |
 | `--config`, `-c` | Config file path |
 | `--port`, `-p` | Server port (default: `3000`) |
 | `--format` | Export format (export only) |

package/dist/bin/tokvista.js

@@ -3,7 +3,7 @@
 // src/bin/tokvista.ts
 import { spawn } from "child_process";
 import { existsSync, readdirSync } from "fs";
-import { readFile, writeFile } from "fs/promises";
+import { readFile as readFile2, writeFile } from "fs/promises";
 import { createServer } from "http";
 import path from "path";
 import { createInterface } from "readline";
@@ -1088,6 +1088,134 @@ function convertTokenFormat(tokens, targetFormat) {
   }
 }
 
+// src/bin/scanner.ts
+import { readdir, readFile } from "fs/promises";
+import { join, extname } from "path";
+function isRecord7(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isTokenLike5(obj) {
+  return isRecord7(obj) && "value" in obj;
+}
+function extractTokenNames(tokens) {
+  const tokenMap = /* @__PURE__ */ new Map();
+  function walk(node, path2 = []) {
+    if (!isRecord7(node)) return;
+    if (path2.length === 0 && Object.keys(node).some((k) => k.includes("/"))) {
+      Object.values(node).forEach((val) => walk(val, []));
+      return;
+    }
+    if (isTokenLike5(node)) {
+      const tokenPath = path2.join(".");
+      const cssVar = `--${path2.join("-")}`;
+      tokenMap.set(cssVar, tokenPath);
+      return;
+    }
+    Object.entries(node).forEach(([key, val]) => {
+      walk(val, [...path2, key]);
+    });
+  }
+  walk(tokens);
+  return tokenMap;
+}
+async function scanFile(filePath, tokenVars) {
+  const content = await readFile(filePath, "utf8");
+  const lines = content.split("\n");
+  const usedVars = /* @__PURE__ */ new Set();
+  const hardcodedColors = [];
+  const hardcodedSpacing = [];
+  const cssVarPattern = /var\((--[\w-]+)\)/g;
+  const hexColorPattern = /#[0-9A-Fa-f]{3,8}\b/g;
+  const rgbPattern = /rgba?\([^)]+\)/g;
+  const spacingPattern = /\b(\d+(?:\.\d+)?(?:px|rem|em))\b/g;
+  lines.forEach((line, index) => {
+    let match;
+    while ((match = cssVarPattern.exec(line)) !== null) {
+      const varName = match[1];
+      if (tokenVars.has(varName)) {
+        usedVars.add(varName);
+      }
+    }
+    while ((match = hexColorPattern.exec(line)) !== null) {
+      const color = match[0];
+      if (!line.includes("0x") && !line.includes("\\u")) {
+        hardcodedColors.push({ line: index + 1, value: color });
+      }
+    }
+    while ((match = rgbPattern.exec(line)) !== null) {
+      hardcodedColors.push({ line: index + 1, value: match[0] });
+    }
+    if (line.includes("padding") || line.includes("margin") || line.includes("gap") || line.includes("width") || line.includes("height")) {
+      while ((match = spacingPattern.exec(line)) !== null) {
+        hardcodedSpacing.push({ line: index + 1, value: match[1] });
+      }
+    }
+  });
+  return { usedVars, hardcodedColors, hardcodedSpacing };
+}
+async function scanDirectory(dir, tokenVars, extensions) {
+  const usedVars = /* @__PURE__ */ new Set();
+  const hardcodedColors = [];
+  const hardcodedSpacing = [];
+  let filesScanned = 0;
+  async function scan(currentDir) {
+    try {
+      const entries = await readdir(currentDir, { withFileTypes: true });
+      for (const entry of entries) {
+        const fullPath = join(currentDir, entry.name);
+        if (entry.isDirectory()) {
+          if (["node_modules", ".git", "dist", "build", ".next", "coverage"].includes(entry.name)) {
+            continue;
+          }
+          await scan(fullPath);
+        } else if (entry.isFile()) {
+          const ext = extname(entry.name);
+          if (extensions.has(ext)) {
+            const result = await scanFile(fullPath, tokenVars);
+            filesScanned++;
+            result.usedVars.forEach((v) => usedVars.add(v));
+            result.hardcodedColors.forEach((h) => hardcodedColors.push({ file: fullPath, ...h }));
+            result.hardcodedSpacing.forEach((h) => hardcodedSpacing.push({ file: fullPath, ...h }));
+          }
+        }
+      }
+    } catch (error) {
+    }
+  }
+  await scan(dir);
+  return { usedVars, hardcodedColors, hardcodedSpacing, filesScanned };
+}
+async function scanTokenUsage(tokensPath, scanDir, tokens) {
+  const detection = detectTokenFormat(tokens);
+  let normalizedTokens = tokens;
+  if (detection.format !== "token-studio" && detection.format !== "unknown") {
+    normalizedTokens = normalizeTokenFormat(tokens, detection.format);
+  }
+  const tokenMap = extractTokenNames(normalizedTokens);
+  const tokenVars = new Set(tokenMap.keys());
+  const extensions = /* @__PURE__ */ new Set([".css", ".scss", ".sass", ".less", ".tsx", ".jsx", ".ts", ".js", ".vue", ".svelte"]);
+  const scanResult = await scanDirectory(scanDir, tokenVars, extensions);
+  const usedTokens = [];
+  const unusedTokens = [];
+  tokenMap.forEach((tokenPath, cssVar) => {
+    if (scanResult.usedVars.has(cssVar)) {
+      usedTokens.push(tokenPath);
+    } else {
+      unusedTokens.push(tokenPath);
+    }
+  });
+  return {
+    totalTokens: tokenMap.size,
+    usedTokens,
+    unusedTokens,
+    hardcodedColors: scanResult.hardcodedColors.slice(0, 50),
+    // Limit to 50
+    hardcodedSpacing: scanResult.hardcodedSpacing.slice(0, 50),
+    // Limit to 50
+    filesScanned: scanResult.filesScanned
+  };
+}
+
 // src/bin/watcher.ts
 import { watch } from "fs";
 function watchFile(filePath, onChange) {
@@ -1122,6 +1250,8 @@ Usage:
   tokvista validate <tokens.json>
   tokvista diff <old.json> <new.json>
   tokvista convert <tokens.json> --to <w3c|style-dictionary|supernova> [--output <file>]
+  tokvista build <tokens.json> --output-dir <dir> [--skip-validation]
+  tokvista scan <directory> [--tokens tokens.json]
 
 Arguments:
   tokens.json       Path to your tokens file (overrides config.tokens)
@@ -1257,6 +1387,12 @@ function parseArgs(args) {
   if (args[0] === "convert") {
     return parseConvertArgs(args.slice(1));
   }
+  if (args[0] === "build") {
+    return parseBuildArgs(args.slice(1));
+  }
+  if (args[0] === "scan") {
+    return parseScanArgs(args.slice(1));
+  }
   return parseServeArgs(args);
 }
 function parseValidateArgs(args) {
@@ -1353,6 +1489,74 @@ function parseConvertArgs(args) {
   if (!to) throw new Error("--to is required (w3c, style-dictionary, supernova, or token-studio)");
   return { command: "convert", tokenFileArg, to, output };
 }
+function parseBuildArgs(args) {
+  let tokenFileArg;
+  let outputDir;
+  let skipValidation = false;
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index];
+    if (arg === "-h" || arg === "--help") {
+      printHelp();
+      process.exit(0);
+    }
+    if (arg === "--output-dir" || arg === "-o") {
+      const next = args[index + 1];
+      if (!next) throw new Error("Missing value for --output-dir");
+      outputDir = next;
+      index += 1;
+      continue;
+    }
+    if (arg.startsWith("--output-dir=")) {
+      outputDir = arg.slice("--output-dir=".length);
+      continue;
+    }
+    if (arg === "--skip-validation") {
+      skipValidation = true;
+      continue;
+    }
+    if (arg.startsWith("-")) {
+      throw new Error(`Unknown option: ${arg}`);
+    }
+    if (tokenFileArg) {
+      throw new Error(`Only one token file is supported. Unexpected value: "${arg}"`);
+    }
+    tokenFileArg = arg;
+  }
+  if (!tokenFileArg) throw new Error("Token file is required for build");
+  if (!outputDir) throw new Error("--output-dir is required");
+  return { command: "build", tokenFileArg, outputDir, skipValidation };
+}
+function parseScanArgs(args) {
+  let scanDir;
+  let tokenFileArg;
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index];
+    if (arg === "-h" || arg === "--help") {
+      printHelp();
+      process.exit(0);
+    }
+    if (arg === "--tokens") {
+      const next = args[index + 1];
+      if (!next) throw new Error("Missing value for --tokens");
+      tokenFileArg = next;
+      index += 1;
+      continue;
+    }
+    if (arg.startsWith("--tokens=")) {
+      tokenFileArg = arg.slice("--tokens=".length);
+      continue;
+    }
+    if (arg.startsWith("-")) {
+      throw new Error(`Unknown option: ${arg}`);
+    }
+    if (scanDir) {
+      throw new Error(`Only one directory is supported. Unexpected value: "${arg}"`);
+    }
+    scanDir = arg;
+  }
+  if (!scanDir) throw new Error("Directory to scan is required");
+  return { command: "scan", scanDir, tokenFileArg };
+}
 function parseExportArgs(args) {
   let tokenFileArg;
   let format;
@@ -1419,7 +1623,7 @@ async function resolveDefaultInitTitle(cwd) {
     return "My Design System";
   }
   try {
-    const raw = await readFile(packageJsonPath, "utf8");
+    const raw = await readFile2(packageJsonPath, "utf8");
     const parsed = JSON.parse(raw);
     if (typeof parsed.name === "string") {
       return formatTitleFromPackageName(parsed.name);
@@ -1659,7 +1863,7 @@ async function resolveLogoForRuntime(logoPathValue, configDir, cwd) {
   if (!existsSync(resolvedLogoPath)) {
     throw new Error(`Logo file not found: ${resolvedLogoPath}`);
   }
-  const content = await readFile(resolvedLogoPath);
+  const content = await readFile2(resolvedLogoPath);
   const mimeType = toDataUrlMimeType(resolvedLogoPath);
   return `data:${mimeType};base64,${content.toString("base64")}`;
 }
@@ -1747,7 +1951,7 @@ async function loadConfigFromFile(configPath) {
   const sourceLabel = path.basename(configPath);
   const extension = path.extname(configPath).toLowerCase();
   if (extension === ".json") {
-    const raw = await readFile(configPath, "utf8");
+    const raw = await readFile2(configPath, "utf8");
     try {
       return normalizeConfigObject(JSON.parse(raw), sourceLabel);
     } catch (error) {
@@ -1755,7 +1959,7 @@ async function loadConfigFromFile(configPath) {
     }
   }
   if (extension === ".ts") {
-    const raw = await readFile(configPath, "utf8");
+    const raw = await readFile2(configPath, "utf8");
     const parsed = parseTsConfigSource(raw, sourceLabel);
     return normalizeConfigObject(parsed, sourceLabel);
   }
@@ -1896,7 +2100,7 @@ function openBrowser(url) {
   });
 }
 async function readTokens(tokenPath) {
-  const raw = await readFile(tokenPath, "utf8");
+  const raw = await readFile2(tokenPath, "utf8");
   try {
     const parsed = JSON.parse(raw);
     if (!parsed || typeof parsed !== "object") {
@@ -1917,8 +2121,8 @@ async function runServeCommand(cwd, options) {
   }
   const runtimeConfig = await buildRuntimeConfig(config, configPath, cwd);
   const [css, appBundle] = await Promise.all([
-    readFile(resolveDistAsset("styles.css"), "utf8"),
-    readFile(resolveDistAsset("cli/browser.js"), "utf8")
+    readFile2(resolveDistAsset("styles.css"), "utf8"),
+    readFile2(resolveDistAsset("cli/browser.js"), "utf8")
   ]);
   let cachedTokens = await readTokens(resolvedTokenPath);
   const getHtml = () => buildHtml(cachedTokens, runtimeConfig, css, appBundle, options.watch);
@@ -2125,6 +2329,106 @@ async function runConvertCommand(cwd, options) {
     console.log(output);
   }
 }
+async function runBuildCommand(cwd, options) {
+  const resolvedTokenPath = path.resolve(cwd, options.tokenFileArg);
+  const outputDir = path.resolve(cwd, options.outputDir);
+  if (!existsSync(resolvedTokenPath)) {
+    throw new Error(`Token file not found: ${resolvedTokenPath}`);
+  }
+  if (!existsSync(outputDir)) {
+    await import("fs/promises").then((fs) => fs.mkdir(outputDir, { recursive: true }));
+  }
+  const tokens = await readTokens(resolvedTokenPath);
+  if (!options.skipValidation) {
+    console.log("\nValidating tokens...");
+    const result = validateTokens(tokens);
+    if (!result.valid) {
+      console.log(`\u274C Found ${result.errors.length} errors`);
+      result.errors.slice(0, 5).forEach((err) => {
+        console.log(`  ${err.path}: ${err.message}`);
+      });
+      if (result.errors.length > 5) {
+        console.log(`  ... and ${result.errors.length - 5} more errors`);
+      }
+      throw new Error("Validation failed. Fix errors or use --skip-validation");
+    }
+    console.log("\u2705 Validation passed\n");
+  }
+  console.log("Building tokens...");
+  const formats = [
+    { name: "CSS", ext: "css", generator: generateCSS },
+    { name: "SCSS", ext: "scss", generator: generateSCSS },
+    { name: "JavaScript", ext: "js", generator: generateJS },
+    { name: "Tailwind", ext: "tailwind.config.js", generator: generateTailwind }
+  ];
+  for (const format of formats) {
+    const content = format.generator(tokens);
+    const filename = format.ext.includes(".") ? format.ext : `tokens.${format.ext}`;
+    const outputPath = path.join(outputDir, filename);
+    await writeFile(outputPath, content, "utf8");
+    console.log(`  \u2713 ${format.name} \u2192 ${filename}`);
+  }
+  console.log(`
+\u2705 Build complete: ${outputDir}
+`);
+}
+async function runScanCommand(cwd, options) {
+  const scanDir = path.resolve(cwd, options.scanDir);
+  const tokenPath = options.tokenFileArg ? path.resolve(cwd, options.tokenFileArg) : path.resolve(cwd, "tokens.json");
+  if (!existsSync(scanDir)) {
+    throw new Error(`Directory not found: ${scanDir}`);
+  }
+  if (!existsSync(tokenPath)) {
+    throw new Error(`Token file not found: ${tokenPath}`);
+  }
+  const tokens = await readTokens(tokenPath);
+  console.log(`
+Scanning ${scanDir} for token usage...
+`);
+  const result = await scanTokenUsage(tokenPath, scanDir, tokens);
+  const usagePercent = (result.usedTokens.length / result.totalTokens * 100).toFixed(1);
+  console.log(`\u{1F4CA} Token Usage Report
+`);
+  console.log(`Files scanned: ${result.filesScanned}`);
+  console.log(`Total tokens: ${result.totalTokens}`);
+  console.log(`Used tokens: ${result.usedTokens.length} (${usagePercent}%)`);
+  console.log(`Unused tokens: ${result.unusedTokens.length}
+`);
+  if (result.unusedTokens.length > 0) {
+    console.log(`\u26A0\uFE0F  Unused Tokens (safe to remove):`);
+    result.unusedTokens.slice(0, 20).forEach((token) => {
+      console.log(`  - ${token}`);
+    });
+    if (result.unusedTokens.length > 20) {
+      console.log(`  ... and ${result.unusedTokens.length - 20} more`);
+    }
+    console.log("");
+  }
+  if (result.hardcodedColors.length > 0) {
+    console.log(`\u{1F3A8} Hardcoded Colors (should use tokens):`);
+    result.hardcodedColors.slice(0, 10).forEach(({ file, line, value }) => {
+      const relPath = path.relative(cwd, file);
+      console.log(`  ${relPath}:${line} - ${value}`);
+    });
+    if (result.hardcodedColors.length > 10) {
+      console.log(`  ... and ${result.hardcodedColors.length - 10} more`);
+    }
+    console.log("");
+  }
+  if (result.hardcodedSpacing.length > 0) {
+    console.log(`\u{1F4CF} Hardcoded Spacing (should use tokens):`);
+    result.hardcodedSpacing.slice(0, 10).forEach(({ file, line, value }) => {
+      const relPath = path.relative(cwd, file);
+      console.log(`  ${relPath}:${line} - ${value}`);
+    });
+    if (result.hardcodedSpacing.length > 10) {
+      console.log(`  ... and ${result.hardcodedSpacing.length - 10} more`);
+    }
+    console.log("");
+  }
+  console.log(`\u2705 Scan complete
+`);
+}
 async function main() {
   try {
     const options = parseArgs(process.argv.slice(2));
@@ -2152,6 +2456,14 @@ async function main() {
       await runConvertCommand(cwd, options);
       return;
     }
+    if (options.command === "build") {
+      await runBuildCommand(cwd, options);
+      return;
+    }
+    if (options.command === "scan") {
+      await runScanCommand(cwd, options);
+      return;
+    }
     await runServeCommand(cwd, options);
   } catch (error) {
     console.error(error.message);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tokvista",
-  "version": "1.9.0",
+  "version": "1.11.0",
   "description": "Interactive visual documentation for design tokens.",
   "type": "module",
   "main": "./dist/index.cjs",