tokvista 1.10.0 → 1.11.0

package/dist/bin/tokvista.js

@@ -3,7 +3,7 @@
 // src/bin/tokvista.ts
 import { spawn } from "child_process";
 import { existsSync, readdirSync } from "fs";
-import { readFile, writeFile } from "fs/promises";
+import { readFile as readFile2, writeFile } from "fs/promises";
 import { createServer } from "http";
 import path from "path";
 import { createInterface } from "readline";
@@ -1088,6 +1088,134 @@ function convertTokenFormat(tokens, targetFormat) {
   }
 }
 
+// src/bin/scanner.ts
+import { readdir, readFile } from "fs/promises";
+import { join, extname } from "path";
+function isRecord7(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isTokenLike5(obj) {
+  return isRecord7(obj) && "value" in obj;
+}
+function extractTokenNames(tokens) {
+  const tokenMap = /* @__PURE__ */ new Map();
+  function walk(node, path2 = []) {
+    if (!isRecord7(node)) return;
+    if (path2.length === 0 && Object.keys(node).some((k) => k.includes("/"))) {
+      Object.values(node).forEach((val) => walk(val, []));
+      return;
+    }
+    if (isTokenLike5(node)) {
+      const tokenPath = path2.join(".");
+      const cssVar = `--${path2.join("-")}`;
+      tokenMap.set(cssVar, tokenPath);
+      return;
+    }
+    Object.entries(node).forEach(([key, val]) => {
+      walk(val, [...path2, key]);
+    });
+  }
+  walk(tokens);
+  return tokenMap;
+}
+async function scanFile(filePath, tokenVars) {
+  const content = await readFile(filePath, "utf8");
+  const lines = content.split("\n");
+  const usedVars = /* @__PURE__ */ new Set();
+  const hardcodedColors = [];
+  const hardcodedSpacing = [];
+  const cssVarPattern = /var\((--[\w-]+)\)/g;
+  const hexColorPattern = /#[0-9A-Fa-f]{3,8}\b/g;
+  const rgbPattern = /rgba?\([^)]+\)/g;
+  const spacingPattern = /\b(\d+(?:\.\d+)?(?:px|rem|em))\b/g;
+  lines.forEach((line, index) => {
+    let match;
+    while ((match = cssVarPattern.exec(line)) !== null) {
+      const varName = match[1];
+      if (tokenVars.has(varName)) {
+        usedVars.add(varName);
+      }
+    }
+    while ((match = hexColorPattern.exec(line)) !== null) {
+      const color = match[0];
+      if (!line.includes("0x") && !line.includes("\\u")) {
+        hardcodedColors.push({ line: index + 1, value: color });
+      }
+    }
+    while ((match = rgbPattern.exec(line)) !== null) {
+      hardcodedColors.push({ line: index + 1, value: match[0] });
+    }
+    if (line.includes("padding") || line.includes("margin") || line.includes("gap") || line.includes("width") || line.includes("height")) {
+      while ((match = spacingPattern.exec(line)) !== null) {
+        hardcodedSpacing.push({ line: index + 1, value: match[1] });
+      }
+    }
+  });
+  return { usedVars, hardcodedColors, hardcodedSpacing };
+}
+async function scanDirectory(dir, tokenVars, extensions) {
+  const usedVars = /* @__PURE__ */ new Set();
+  const hardcodedColors = [];
+  const hardcodedSpacing = [];
+  let filesScanned = 0;
+  async function scan(currentDir) {
+    try {
+      const entries = await readdir(currentDir, { withFileTypes: true });
+      for (const entry of entries) {
+        const fullPath = join(currentDir, entry.name);
+        if (entry.isDirectory()) {
+          if (["node_modules", ".git", "dist", "build", ".next", "coverage"].includes(entry.name)) {
+            continue;
+          }
+          await scan(fullPath);
+        } else if (entry.isFile()) {
+          const ext = extname(entry.name);
+          if (extensions.has(ext)) {
+            const result = await scanFile(fullPath, tokenVars);
+            filesScanned++;
+            result.usedVars.forEach((v) => usedVars.add(v));
+            result.hardcodedColors.forEach((h) => hardcodedColors.push({ file: fullPath, ...h }));
+            result.hardcodedSpacing.forEach((h) => hardcodedSpacing.push({ file: fullPath, ...h }));
+          }
+        }
+      }
+    } catch (error) {
+    }
+  }
+  await scan(dir);
+  return { usedVars, hardcodedColors, hardcodedSpacing, filesScanned };
+}
+async function scanTokenUsage(tokensPath, scanDir, tokens) {
+  const detection = detectTokenFormat(tokens);
+  let normalizedTokens = tokens;
+  if (detection.format !== "token-studio" && detection.format !== "unknown") {
+    normalizedTokens = normalizeTokenFormat(tokens, detection.format);
+  }
+  const tokenMap = extractTokenNames(normalizedTokens);
+  const tokenVars = new Set(tokenMap.keys());
+  const extensions = /* @__PURE__ */ new Set([".css", ".scss", ".sass", ".less", ".tsx", ".jsx", ".ts", ".js", ".vue", ".svelte"]);
+  const scanResult = await scanDirectory(scanDir, tokenVars, extensions);
+  const usedTokens = [];
+  const unusedTokens = [];
+  tokenMap.forEach((tokenPath, cssVar) => {
+    if (scanResult.usedVars.has(cssVar)) {
+      usedTokens.push(tokenPath);
+    } else {
+      unusedTokens.push(tokenPath);
+    }
+  });
+  return {
+    totalTokens: tokenMap.size,
+    usedTokens,
+    unusedTokens,
+    hardcodedColors: scanResult.hardcodedColors.slice(0, 50),
+    // Limit to 50
+    hardcodedSpacing: scanResult.hardcodedSpacing.slice(0, 50),
+    // Limit to 50
+    filesScanned: scanResult.filesScanned
+  };
+}
+
 // src/bin/watcher.ts
 import { watch } from "fs";
 function watchFile(filePath, onChange) {
@@ -1123,6 +1251,7 @@ Usage:
   tokvista diff <old.json> <new.json>
   tokvista convert <tokens.json> --to <w3c|style-dictionary|supernova> [--output <file>]
   tokvista build <tokens.json> --output-dir <dir> [--skip-validation]
+  tokvista scan <directory> [--tokens tokens.json]
 
 Arguments:
   tokens.json       Path to your tokens file (overrides config.tokens)
@@ -1261,6 +1390,9 @@ function parseArgs(args) {
   if (args[0] === "build") {
     return parseBuildArgs(args.slice(1));
   }
+  if (args[0] === "scan") {
+    return parseScanArgs(args.slice(1));
+  }
   return parseServeArgs(args);
 }
 function parseValidateArgs(args) {
@@ -1394,6 +1526,37 @@ function parseBuildArgs(args) {
   if (!outputDir) throw new Error("--output-dir is required");
   return { command: "build", tokenFileArg, outputDir, skipValidation };
 }
+function parseScanArgs(args) {
+  let scanDir;
+  let tokenFileArg;
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index];
+    if (arg === "-h" || arg === "--help") {
+      printHelp();
+      process.exit(0);
+    }
+    if (arg === "--tokens") {
+      const next = args[index + 1];
+      if (!next) throw new Error("Missing value for --tokens");
+      tokenFileArg = next;
+      index += 1;
+      continue;
+    }
+    if (arg.startsWith("--tokens=")) {
+      tokenFileArg = arg.slice("--tokens=".length);
+      continue;
+    }
+    if (arg.startsWith("-")) {
+      throw new Error(`Unknown option: ${arg}`);
+    }
+    if (scanDir) {
+      throw new Error(`Only one directory is supported. Unexpected value: "${arg}"`);
+    }
+    scanDir = arg;
+  }
+  if (!scanDir) throw new Error("Directory to scan is required");
+  return { command: "scan", scanDir, tokenFileArg };
+}
 function parseExportArgs(args) {
   let tokenFileArg;
   let format;
@@ -1460,7 +1623,7 @@ async function resolveDefaultInitTitle(cwd) {
     return "My Design System";
   }
   try {
-    const raw = await readFile(packageJsonPath, "utf8");
+    const raw = await readFile2(packageJsonPath, "utf8");
     const parsed = JSON.parse(raw);
     if (typeof parsed.name === "string") {
       return formatTitleFromPackageName(parsed.name);
@@ -1700,7 +1863,7 @@ async function resolveLogoForRuntime(logoPathValue, configDir, cwd) {
   if (!existsSync(resolvedLogoPath)) {
     throw new Error(`Logo file not found: ${resolvedLogoPath}`);
   }
-  const content = await readFile(resolvedLogoPath);
+  const content = await readFile2(resolvedLogoPath);
   const mimeType = toDataUrlMimeType(resolvedLogoPath);
   return `data:${mimeType};base64,${content.toString("base64")}`;
 }
@@ -1788,7 +1951,7 @@ async function loadConfigFromFile(configPath) {
   const sourceLabel = path.basename(configPath);
   const extension = path.extname(configPath).toLowerCase();
   if (extension === ".json") {
-    const raw = await readFile(configPath, "utf8");
+    const raw = await readFile2(configPath, "utf8");
     try {
       return normalizeConfigObject(JSON.parse(raw), sourceLabel);
     } catch (error) {
@@ -1796,7 +1959,7 @@ async function loadConfigFromFile(configPath) {
     }
   }
   if (extension === ".ts") {
-    const raw = await readFile(configPath, "utf8");
+    const raw = await readFile2(configPath, "utf8");
     const parsed = parseTsConfigSource(raw, sourceLabel);
     return normalizeConfigObject(parsed, sourceLabel);
   }
@@ -1937,7 +2100,7 @@ function openBrowser(url) {
   });
 }
 async function readTokens(tokenPath) {
-  const raw = await readFile(tokenPath, "utf8");
+  const raw = await readFile2(tokenPath, "utf8");
   try {
     const parsed = JSON.parse(raw);
     if (!parsed || typeof parsed !== "object") {
@@ -1958,8 +2121,8 @@ async function runServeCommand(cwd, options) {
   }
   const runtimeConfig = await buildRuntimeConfig(config, configPath, cwd);
   const [css, appBundle] = await Promise.all([
-    readFile(resolveDistAsset("styles.css"), "utf8"),
-    readFile(resolveDistAsset("cli/browser.js"), "utf8")
+    readFile2(resolveDistAsset("styles.css"), "utf8"),
+    readFile2(resolveDistAsset("cli/browser.js"), "utf8")
   ]);
   let cachedTokens = await readTokens(resolvedTokenPath);
   const getHtml = () => buildHtml(cachedTokens, runtimeConfig, css, appBundle, options.watch);
@@ -2209,6 +2372,63 @@ async function runBuildCommand(cwd, options) {
 \u2705 Build complete: ${outputDir}
 `);
 }
+async function runScanCommand(cwd, options) {
+  const scanDir = path.resolve(cwd, options.scanDir);
+  const tokenPath = options.tokenFileArg ? path.resolve(cwd, options.tokenFileArg) : path.resolve(cwd, "tokens.json");
+  if (!existsSync(scanDir)) {
+    throw new Error(`Directory not found: ${scanDir}`);
+  }
+  if (!existsSync(tokenPath)) {
+    throw new Error(`Token file not found: ${tokenPath}`);
+  }
+  const tokens = await readTokens(tokenPath);
+  console.log(`
+Scanning ${scanDir} for token usage...
+`);
+  const result = await scanTokenUsage(tokenPath, scanDir, tokens);
+  const usagePercent = (result.usedTokens.length / result.totalTokens * 100).toFixed(1);
+  console.log(`\u{1F4CA} Token Usage Report
+`);
+  console.log(`Files scanned: ${result.filesScanned}`);
+  console.log(`Total tokens: ${result.totalTokens}`);
+  console.log(`Used tokens: ${result.usedTokens.length} (${usagePercent}%)`);
+  console.log(`Unused tokens: ${result.unusedTokens.length}
+`);
+  if (result.unusedTokens.length > 0) {
+    console.log(`\u26A0\uFE0F  Unused Tokens (safe to remove):`);
+    result.unusedTokens.slice(0, 20).forEach((token) => {
+      console.log(`  - ${token}`);
+    });
+    if (result.unusedTokens.length > 20) {
+      console.log(`  ... and ${result.unusedTokens.length - 20} more`);
+    }
+    console.log("");
+  }
+  if (result.hardcodedColors.length > 0) {
+    console.log(`\u{1F3A8} Hardcoded Colors (should use tokens):`);
+    result.hardcodedColors.slice(0, 10).forEach(({ file, line, value }) => {
+      const relPath = path.relative(cwd, file);
+      console.log(`  ${relPath}:${line} - ${value}`);
+    });
+    if (result.hardcodedColors.length > 10) {
+      console.log(`  ... and ${result.hardcodedColors.length - 10} more`);
+    }
+    console.log("");
+  }
+  if (result.hardcodedSpacing.length > 0) {
+    console.log(`\u{1F4CF} Hardcoded Spacing (should use tokens):`);
+    result.hardcodedSpacing.slice(0, 10).forEach(({ file, line, value }) => {
+      const relPath = path.relative(cwd, file);
+      console.log(`  ${relPath}:${line} - ${value}`);
+    });
+    if (result.hardcodedSpacing.length > 10) {
+      console.log(`  ... and ${result.hardcodedSpacing.length - 10} more`);
+    }
+    console.log("");
+  }
+  console.log(`\u2705 Scan complete
+`);
+}
 async function main() {
   try {
     const options = parseArgs(process.argv.slice(2));
@@ -2240,6 +2460,10 @@ async function main() {
       await runBuildCommand(cwd, options);
       return;
     }
+    if (options.command === "scan") {
+      await runScanCommand(cwd, options);
+      return;
+    }
     await runServeCommand(cwd, options);
   } catch (error) {
     console.error(error.message);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tokvista",
-  "version": "1.10.0",
+  "version": "1.11.0",
   "description": "Interactive visual documentation for design tokens.",
   "type": "module",
   "main": "./dist/index.cjs",