prism-design 2.13.0

package/scripts/tokenize.ts

@@ -0,0 +1,988 @@
+/**
+ * Prism — Tokenizer
+ *
+ * Convertit raw-css.json en tokens.json structuré et normalisé.
+ * Chaque token vient EXCLUSIVEMENT de l'extraction — aucune valeur inventée.
+ */
+
+import { readFile, writeFile } from 'fs/promises';
+import { join } from 'path';
+
+interface DesignTokens {
+  meta: {
+    source: string;
+    domain: string;
+    extractedAt: string;
+    tokenizedAt: string;
+  };
+  colors: {
+    background: { primary: string; secondary: string; tertiary: string };
+    text: { primary: string; secondary: string; muted: string };
+    accent: { primary: string | null; secondary: string | null };
+    border: string;
+    shadow: string;
+    semantic?: { error?: string; success?: string; warning?: string; info?: string };
+  };
+  typography: {
+    fontFamily: { primary: string; secondary: string; mono: string };
+    fontSize: { xs: string; sm: string; base: string; lg: string; xl: string; '2xl': string; '3xl': string; '4xl': string };
+    fontWeight: { normal: string; medium: string; semibold: string; bold: string };
+    lineHeight: { tight: string; normal: string; relaxed: string };
+    letterSpacing: { tight: string; normal: string; wide: string };
+  };
+  spacing: {
+    xxs: string; xs: string; sm: string; md: string; base: string; lg: string; xl: string; '2xl': string; '3xl': string;
+  };
+  borderRadius: {
+    none: string; xs: string; sm: string; md: string; lg: string; xl: string; full: string;
+  };
+  shadows: Record<string, string>;
+  transitions: Record<string, string>;
+  layout: {
+    maxWidth: string;
+    headerHeight: string;
+    sidebarWidth: string;
+    gap: string;
+    containerPadding: string;
+  };
+  cssCustomProperties: Record<string, string>;
+}
+
+import {
+  parseRgb as parseRgbShared,
+  luminanceSimple as luminance,
+  parsePixels,
+  cleanColorValue,
+} from './shared/css-helpers.js';
+
+/** tokenize variant: gère CSS shorthand multi-value via first-color split */
+function parseRgb(color: string): [number, number, number] | null {
+  if (!color) return null;
+  const firstColor = color.split(/\s+(?=rgb)/)[0].trim();
+  return parseRgbShared(firstColor);
+}
+
+function sortByPixels(values: string[]): string[] {
+  return [...values].filter(v => v.includes('px')).sort((a, b) => parsePixels(a) - parsePixels(b));
+}
+
+function pickClosest(values: string[], targetPx: number): string {
+  if (values.length === 0) return `${targetPx}px`;
+  let best = values[0];
+  let bestDist = Math.abs(parsePixels(best) - targetPx);
+  for (const v of values) {
+    const dist = Math.abs(parsePixels(v) - targetPx);
+    if (dist < bestDist) { best = v; bestDist = dist; }
+  }
+  return best;
+}
+
+/**
+ * Build a distinct spacing scale: for each target, pick the closest value
+ * BUT prefer values not already assigned to a smaller step (ensures md ≠ lg ≠ xl).
+ * If no distinct value exists (sparse data), synthesize a fallback via target × ratio.
+ */
+function pickDistinctScale(values: string[], targets: number[]): string[] {
+  const distinct = Array.from(new Set(values)).sort((a, b) => parsePixels(a) - parsePixels(b));
+  const result: string[] = [];
+  const used = new Set<string>();
+
+  for (let i = 0; i < targets.length; i++) {
+    const target = targets[i];
+    // Find unused values, pick the closest to target
+    const available = distinct.filter(v => !used.has(v));
+    if (available.length === 0) {
+      // Out of distinct values: synthesize from target (8px scale: 4, 8, 12, 16, 24, 32, 48, 64)
+      result.push(`${target}px`);
+      continue;
+    }
+    let best = available[0];
+    let bestDist = Math.abs(parsePixels(best) - target);
+    for (const v of available) {
+      const dist = Math.abs(parsePixels(v) - target);
+      // Prefer values ≥ previous step (preserves monotonic scale)
+      if (i > 0) {
+        const prevPx = parsePixels(result[i - 1]);
+        if (parsePixels(v) < prevPx) continue;
+      }
+      if (dist < bestDist) { best = v; bestDist = dist; }
+    }
+    // Validate: if best is < previous step (regression), synthesize from target
+    if (i > 0 && parsePixels(best) <= parsePixels(result[i - 1])) {
+      // Try next-larger value in available
+      const next = available.find(v => parsePixels(v) > parsePixels(result[i - 1]));
+      if (next) { result.push(next); used.add(next); continue; }
+      result.push(`${target}px`);
+      continue;
+    }
+    result.push(best);
+    used.add(best);
+  }
+  // Phase 5.2.4 — Final monotonicity sweep: walk result, replace any step that's ≤ previous
+  // with max(target, prevPx+8). Using just targets[i] was the bug: if target < prevPx the sweep
+  // left the sequence non-monotone (Stripe 2xl=71→3xl=64, switchcollective xl=72→2xl=48 cases).
+  for (let i = 1; i < result.length; i++) {
+    if (parsePixels(result[i]) <= parsePixels(result[i - 1])) {
+      const prevPx = parsePixels(result[i - 1]);
+      result[i] = `${Math.max(targets[i], prevPx + 8)}px`;
+    }
+  }
+  return result;
+}
+
+function normalizeLineHeight(lineHeight: string | undefined, fontSize: string | undefined): string | null {
+  if (!lineHeight) return null;
+  // Already a ratio (no unit or unitless number)
+  if (!lineHeight.includes('px') && !lineHeight.includes('em')) {
+    const num = parseFloat(lineHeight);
+    return isNaN(num) ? null : num.toFixed(2).replace(/\.?0+$/, '');
+  }
+  // Pixel value — divide by fontSize to get ratio
+  const lhPx = parsePixels(lineHeight);
+  const fsPx = fontSize ? parsePixels(fontSize) : 0;
+  if (lhPx > 0 && fsPx > 0) {
+    const ratio = lhPx / fsPx;
+    // Round to 2 decimal places, strip trailing zeros
+    return ratio.toFixed(2).replace(/\.?0+$/, '');
+  }
+  return null;
+}
+
+function getSaturation(rgb: [number, number, number]): number {
+  return Math.max(...rgb) - Math.min(...rgb);
+}
+
+/**
+ * Resolve container max-width. CSS computed `main.maxWidth` is often `none`
+ * because the limit lives on a child container or in a CSS variable.
+ * Strategy: 1) trust element if explicit, 2) scan CSS vars for max-width
+ * tokens (homepage > page > container > content > prose priority), 3) eval
+ * calc() expressions if encountered.
+ */
+function resolveMaxWidth(elements: any, cssVars: Record<string, string>): string {
+  const main = elements.main?.styles?.maxWidth;
+  if (main && main !== 'none' && main !== 'unset' && main !== 'auto') return main;
+
+  // Try common semantic container selectors first
+  for (const sel of ['hero', 'header', 'footer']) {
+    const w = elements[sel]?.styles?.maxWidth;
+    if (w && w !== 'none' && w !== 'unset' && w !== 'auto') return evalCalc(w);
+  }
+
+  // Scan CSS vars by priority — most-specific brand names win
+  const priorityOrder: RegExp[] = [
+    /homepage-?(max-?)?width/i,
+    /page-?(max-?)?width/i,
+    /site-?(max-?)?width/i,
+    /container-?(max-?)?width/i,
+    /content-?(max-?)?width/i,
+    /max-?width/i,
+  ];
+  for (const re of priorityOrder) {
+    for (const [key, value] of Object.entries(cssVars)) {
+      if (re.test(key) && typeof value === 'string' && value.trim()) {
+        const resolved = evalCalc(value.trim());
+        if (resolved && resolved !== 'none') return resolved;
+      }
+    }
+  }
+
+  // Last resort
+  return '1200px';
+}
+
+/** Evaluate a simple CSS calc(X + Y * Z) into a px value if possible. */
+function evalCalc(value: string): string {
+  if (!value.includes('calc(')) return value;
+  const inner = value.match(/calc\(([^()]+)\)/)?.[1];
+  if (!inner) return value;
+  try {
+    // Strip "px" units, evaluate, re-append "px"
+    const expr = inner.replace(/px/g, '').replace(/\s+/g, ' ');
+    // Defensive: only allow digits, +, -, *, /, ., (, ), spaces
+    if (!/^[\d+\-*/.()\s]+$/.test(expr)) return value;
+    const result = Function(`"use strict"; return (${expr})`)();
+    if (typeof result === 'number' && isFinite(result) && result > 0) return `${Math.round(result)}px`;
+  } catch { /* fallthrough */ }
+  return value;
+}
+
+/**
+ * Pick a text.secondary that is GUARANTEED visually distinct from primary.
+ * Fix (audit-2026-05-28): "text.secondary dégénère à white/black 100% des cas"
+ * because classified.text[1] often equals bodyColor (no real second tier found).
+ *
+ * Strategy:
+ * 1. Walk classified.text for a value with luminance diff > 0.12 from primary
+ * 2. If none found, synthesize via rgba alpha (0.65 for opaque colors)
+ */
+function pickDistinctTextSecondary(
+  textColors: string[],
+  primary: string,
+  isDark: boolean
+): string {
+  const primaryRgb = parseRgb(primary);
+  if (!primaryRgb) return textColors[1] || textColors[0] || primary;
+  const primaryLum = luminance(primaryRgb);
+
+  for (const candidate of textColors) {
+    const candRgb = parseRgb(candidate);
+    if (!candRgb) continue;
+    const candLum = luminance(candRgb);
+    if (Math.abs(candLum - primaryLum) > 0.12) {
+      return candidate;
+    }
+  }
+
+  // Synthesize via alpha blend (preserves brand color tone)
+  const [r, g, b] = primaryRgb;
+  return `rgba(${r}, ${g}, ${b}, 0.65)`;
+}
+
+/**
+ * Deduplicate colors perceptually — merge near-identical colors.
+ * Uses simple RGB distance (good enough for dedup); cluster representative = first occurrence.
+ * Threshold ~15 means colors with ΔRGB < 15 are considered "same color" (build noise, sub-pixel).
+ */
+function dedupePerceptual(colors: string[], threshold = 12): string[] {
+  const reps: Array<{ raw: string; rgb: [number, number, number] }> = [];
+  for (const c of colors) {
+    const rgb = parseRgb(c);
+    if (!rgb) continue;
+    let isDup = false;
+    for (const rep of reps) {
+      const d = Math.sqrt(
+        Math.pow(rgb[0] - rep.rgb[0], 2) +
+        Math.pow(rgb[1] - rep.rgb[1], 2) +
+        Math.pow(rgb[2] - rep.rgb[2], 2)
+      );
+      if (d < threshold) { isDup = true; break; }
+    }
+    if (!isDup) reps.push({ raw: c, rgb: [rgb[0], rgb[1], rgb[2]] });
+  }
+  return reps.map(r => r.raw);
+}
+
+function classifyColors(
+  colors: string[],
+  bodyBgColor?: string,
+): { backgrounds: string[]; text: string[]; accents: string[]; border: string[] } {
+  const backgrounds: string[] = [];
+  const text: string[] = [];
+  const accents: string[] = [];
+  const border: string[] = [];
+
+  const seen = new Set<string>();
+
+  // Detect if site is dark-mode from body bg
+  const bodyBgRgb = bodyBgColor ? parseRgb(bodyBgColor) : null;
+  const isDark = bodyBgRgb ? luminance(bodyBgRgb) < 0.3 : false;
+
+  for (const c of colors) {
+    const norm = c.replace(/\s+/g, '');
+    if (seen.has(norm)) continue;
+    seen.add(norm);
+
+    const rgb = parseRgb(c);
+    if (!rgb) continue;
+    const lum = luminance(rgb);
+    const sat = getSaturation(rgb);
+
+    // Saturated colors are always accents regardless of mode
+    if (sat > 50 && lum > 0.1 && lum < 0.9) {
+      accents.push(c);
+      continue;
+    }
+
+    if (isDark) {
+      // Dark site: dark = backgrounds/surfaces, light = text, semi-transparent borders
+      if (lum < 0.1) backgrounds.push(c);           // near-black backgrounds
+      else if (lum < 0.3) backgrounds.push(c);       // dark elevated surfaces
+      else if (lum > 0.75) text.push(c);             // light text on dark bg
+      else if (lum > 0.45 && lum <= 0.75) text.push(c); // muted text (secondary, tertiary)
+      else border.push(c);                            // mid tones = borders/dividers
+    } else {
+      // Light site: light = backgrounds, dark = text, near-white borders
+      if (lum > 0.9) backgrounds.push(c);
+      else if (lum > 0.7) border.push(c);
+      else if (lum < 0.15) text.push(c);
+      // Mid-tones (lum 0.15–0.7): accents ONLY if saturated (sat > 30)
+      // Otherwise neutral grays → borders/dividers, NOT accents
+      else if (sat > 30) accents.push(c);
+      else border.push(c);
+    }
+  }
+
+  return { backgrounds, text, accents, border };
+}
+
+async function tokenize(domain: string): Promise<void> {
+  const baseDir = join(process.cwd(), 'extractions', domain);
+  const rawPath = join(baseDir, 'raw-css.json');
+
+  console.log(`\n🔧 Tokenizing ${domain}...`);
+
+  const raw = JSON.parse(await readFile(rawPath, 'utf-8'));
+  const data = raw.desktop;
+  if (!data) { console.error('No desktop data'); process.exit(1); }
+
+  const { elements, allColors, allFontFamilies, allFontSizes, allBorderRadii, allShadows, allTransitions, cssCustomProperties } = data;
+
+  // ── Colors — detect dark mode before classification ──
+  // Handle transparent body backgrounds (common on Shopify, SPAs)
+  // If body bg is transparent, find the first non-transparent bg from sections/elements
+  function resolveBodyBackground(): string {
+    const rawBg = elements.body?.styles?.backgroundColor;
+    const isTransparent = !rawBg || rawBg === 'rgba(0, 0, 0, 0)' || rawBg === 'transparent';
+    if (!isTransparent) return rawBg;
+
+    // Gradient fallback: extract first color from backgroundImage (e.g. Vercel, Linear dark mode)
+    const bgImage = elements.body?.styles?.backgroundImage ||
+      (elements as any).main?.styles?.backgroundImage;
+    if (bgImage && bgImage !== 'none') {
+      const firstColor = bgImage.match(/rgb[a]?\([^)]+\)/)?.[0];
+      if (firstColor) return firstColor;
+    }
+
+    // CSS var fallback: check common background CSS var names
+    const bgVarKeys = ['--background', '--bg', '--color-background', '--surface', '--page-bg', '--bg-primary'];
+    for (const key of bgVarKeys) {
+      const v = String((cssCustomProperties as Record<string, string>)[key] || '').trim();
+      if (!v) continue;
+      if (parseRgb(v)) return v;
+      const hm = v.match(/^#([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i);
+      if (hm) return v; // return hex as-is, parseRgb caller handles hex
+    }
+
+    // Try other semantic elements for background
+    const candidates = [elements.main, elements.header, elements.hero, elements.card];
+    for (const el of candidates) {
+      const bg = el?.styles?.backgroundColor;
+      if (bg && bg !== 'rgba(0, 0, 0, 0)' && bg !== 'transparent') return bg;
+    }
+
+    // v8-BUG-FIX: When body+html+main+header are ALL transparent, the actual visible bg
+    // is the browser default = WHITE. Do NOT fallback to allColors[0] which is
+    // typically the text color (rgb(0,0,0)) — that would falsely classify the site as dark.
+    // Web standard: transparent body → render on white canvas.
+    // This fixes addictsneakers.com (was wrongly detected as dark mode).
+    return 'rgb(255,255,255)';
+  }
+
+  // v2.6 Fix-1/5 — DIRECTIONAL canvas rescue. A common false-dark: <body> has a dark wrapper color
+  // (notion: #191918) while the actual content sits on white sections. Fix: if body is dark BUT the
+  // dominant rendered section area is light, the visible canvas is light. The reverse is NOT applied —
+  // a light <body> with large dark marketing bands (miro, revolut) is still a light site, so we never
+  // flip light→dark. This rescues notion without regressing miro/revolut (verified).
+  function toRgb(c: string): [number, number, number] | null {
+    return parseRgb(c) ?? (() => {
+      const hm = c.match(/^#([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i);
+      return hm ? [parseInt(hm[1], 16), parseInt(hm[2], 16), parseInt(hm[3], 16)] as [number, number, number] : null;
+    })();
+  }
+  function resolveDominantSectionBg(): string | null {
+    const sections = ((data as any).sections || []) as any[];
+    const areaByBg = new Map<string, number>();
+    for (const sec of sections) {
+      const bg: string = sec?.styles?.backgroundColor || '';
+      if (!parseRgb(bg)) continue;
+      const alphaM = bg.match(/rgba?\([^)]*,\s*([\d.]+)\s*\)/);
+      if (alphaM && parseFloat(alphaM[1]) === 0) continue; // skip transparent
+      const area = (sec?.rect?.height || 0) * (sec?.rect?.width || 0);
+      if (area <= 0) continue;
+      const k = cleanColorValue(bg);
+      areaByBg.set(k, (areaByBg.get(k) || 0) + area);
+    }
+    if (!areaByBg.size) return null;
+    return [...areaByBg.entries()].sort((x, y) => y[1] - x[1])[0][0];
+  }
+  function resolveCanvas(): string {
+    const body = resolveBodyBackground();
+    const bodyRgb = toRgb(body);
+    if (!bodyRgb || luminance(bodyRgb) >= 0.3) return body; // only rescue the dark-body case
+    const dom = resolveDominantSectionBg();
+    if (dom) {
+      const domRgb = toRgb(dom);
+      if (domRgb && luminance(domRgb) >= 0.3) return dom;   // dark body + light content → light canvas
+    }
+    return body;
+  }
+
+  const bodyBg = resolveCanvas();
+  const classified = classifyColors(allColors, bodyBg);
+
+  const bodyColor = elements.body?.styles?.color || classified.text[0] || 'rgb(0,0,0)';
+
+  // Detect dark site — also handle hex values in bodyBg (from CSS var fallback above)
+  const bodyBgRgb: [number, number, number] | null = parseRgb(bodyBg) ?? (() => {
+    const hm = bodyBg.match(/^#([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i);
+    return hm ? [parseInt(hm[1], 16), parseInt(hm[2], 16), parseInt(hm[3], 16)] as [number, number, number] : null;
+  })();
+  const isDarkSite = bodyBgRgb ? luminance(bodyBgRgb) < 0.3 : false;
+
+  // ── Typography ──
+  // Generic/fallback fonts that should never be picked as primary when better options exist
+  const GENERIC_FONTS = new Set([
+    'times new roman', 'times', 'georgia', 'palatino', 'garamond', 'bookman',
+    'serif', 'sans-serif', 'monospace', 'cursive', 'fantasy',
+    'arial', 'helvetica', 'verdana', 'tahoma', 'trebuchet ms', 'comic sans ms',
+    'apple color emoji', 'segoe ui emoji', 'segoe ui symbol', 'noto color emoji',
+    // KaTeX math library fonts — always parasitic, never brand fonts
+    'katex_main', 'katex_ams', 'katex_caligraphic', 'katex_fraktur',
+    'katex_math', 'katex_mathit', 'katex_size1', 'katex_size2', 'katex_size3', 'katex_size4',
+  ]);
+
+  // System/fallback fonts that should only be picked if no custom web font exists
+  const SYSTEM_FONTS = new Set([
+    '-apple-system', 'blinkmacsystemfont', 'system-ui', 'segoe ui',
+    'ui-sans-serif', 'ui-serif', 'ui-monospace',
+    // i18n fallback fonts — lose to any branded web font
+    'noto sans', 'noto serif', 'noto sans math', 'noto sans arabic',
+    'noto sans hebrew', 'noto sans jp', 'noto sans kr', 'noto sans sc',
+  ]);
+
+  // Extract all individual font names from all font-family stacks
+  const allIndividualFonts: string[] = [];
+  for (const stack of allFontFamilies) {
+    const parts = (stack as string).split(',').map((f: string) => f.trim().replace(/"/g, ''));
+    allIndividualFonts.push(...parts);
+  }
+
+  // Also extract the body font specifically — pick the best font from body's stack
+  const bodyFontStack = elements.body?.styles?.fontFamily || '';
+  const bodyFontParts = bodyFontStack.split(',').map((f: string) => f.trim().replace(/"/g, ''));
+
+  function pickBestFont(fontList: string[]): string {
+    // First pass: look for custom web fonts (not generic, not system, not KaTeX)
+    for (const font of fontList) {
+      const lower = font.toLowerCase();
+      if (/^katex_/i.test(font)) continue; // always skip KaTeX math library
+      if (font && !GENERIC_FONTS.has(lower) && !SYSTEM_FONTS.has(lower)) return font;
+    }
+    // Second pass: accept system fonts as fallback
+    for (const font of fontList) {
+      const lower = font.toLowerCase();
+      if (font && !GENERIC_FONTS.has(lower)) return font;
+    }
+    // Last resort
+    return fontList[0] || 'system-ui';
+  }
+
+  // Deduplicate, filter generics, and sort with priority fonts first
+  const seen = new Set<string>();
+  const uniqueFamilies: string[] = [];
+
+  // Start with the body font (most important — sets the page's primary font)
+  let bodyPrimary = pickBestFont(bodyFontParts);
+  // v2.6 Fix-3 — if body resolves to a GENERIC/system font (e.g. miro body = "sans-serif"), the real
+  // brand font lives in heading/other stacks (e.g. Roobert PRO in allFontFamilies). Prefer it so the
+  // primary font isn't a generic placeholder (which then leaks into the YAML + narrative as truth).
+  const isGenericFontName = (f: string) => !f || GENERIC_FONTS.has(f.toLowerCase()) || SYSTEM_FONTS.has(f.toLowerCase());
+  if (isGenericFontName(bodyPrimary)) {
+    for (const stack of allFontFamilies) {
+      const parts = String(stack).split(',').map((f: string) => f.trim().replace(/['"]/g, ''));
+      const real = parts.find((f: string) => f && !isGenericFontName(f) && !/^katex_/i.test(f));
+      if (real) { bodyPrimary = real; break; }
+    }
+  }
+  if (bodyPrimary) {
+    seen.add(bodyPrimary.toLowerCase());
+    uniqueFamilies.push(bodyPrimary);
+  }
+
+  // Then add other meaningful fonts from all stacks
+  for (const font of allIndividualFonts) {
+    const lower = font.toLowerCase();
+    if (!font || seen.has(lower) || GENERIC_FONTS.has(lower)) continue;
+    seen.add(lower);
+    uniqueFamilies.push(font);
+  }
+
+  const sortedSizes = sortByPixels(allFontSizes);
+
+  // ── Spacing — CSS vars first (--spacing-* / --space-*), element padding as fallback ──
+  const spacingValues = new Set<string>();
+  const cssVarsForSpacing = (cssCustomProperties || {}) as Record<string, string>;
+  for (const [key, val] of Object.entries(cssVarsForSpacing)) {
+    if (/^--(spacing|space|size|gap)-/i.test(key) && val) {
+      const v = val.trim();
+      if (v.includes('px') && parsePixels(v) > 0 && parsePixels(v) < 200) {
+        spacingValues.add(v);
+      } else if (v.includes('rem')) {
+        const px = Math.round(parseFloat(v) * 16);
+        if (px > 0 && px < 200) spacingValues.add(`${px}px`);
+      }
+    }
+  }
+  // fallback: element-based padding/margin/gap if CSS vars yielded < 4 values
+  if (spacingValues.size < 4) {
+    for (const el of Object.values(elements)) {
+      if (!el) continue;
+      const e = el as { styles: Record<string, string> };
+      for (const prop of ['padding', 'margin', 'gap']) {
+        const val = e.styles?.[prop];
+        if (val) {
+          val.split(' ').forEach((v: string) => {
+            if (v.includes('px')) {
+              const px = Math.round(parsePixels(v));
+              if (px > 0 && px < 200) spacingValues.add(`${px}px`);
+            }
+          });
+        }
+      }
+    }
+    // Also pull section-level vertical padding (vPad) — premium sites put their
+    // generous whitespace on <section> wrappers, not on the key elements above.
+    const sectionsData = ((data as any).sections || []) as any[];
+    for (const sec of sectionsData) {
+      if (sec?.vPad && typeof sec.vPad === 'number') {
+        const px = Math.round(sec.vPad);
+        if (px > 0 && px < 200) spacingValues.add(`${px}px`);
+      }
+      // Also extract from section styles directly
+      const secPad = sec?.styles?.padding;
+      if (secPad && typeof secPad === 'string') {
+        secPad.split(' ').forEach((v: string) => {
+          if (v.includes('px')) {
+            const px = Math.round(parsePixels(v));
+            if (px > 0 && px < 200) spacingValues.add(`${px}px`);
+          }
+        });
+      }
+    }
+  }
+  const sortedSpacing = sortByPixels([...spacingValues]);
+
+  // ── Border radii — CSS vars first (--corner-radius-* / --radius-*), then extracted values ──
+  const radiusFromVars = new Set<string>();
+  for (const [key, val] of Object.entries(cssVarsForSpacing)) {
+    if (/^--(corner-radius|radius|rounded)-/i.test(key) && val) {
+      const v = val.trim();
+      if (v.includes('px') && parsePixels(v) >= 0 && parsePixels(v) < 9999) {
+        radiusFromVars.add(v);
+      } else if (v === '50%' || v === '100%' || v.includes('%')) {
+        radiusFromVars.add(v);
+      }
+    }
+  }
+  // ALWAYS merge allBorderRadii (actual rendered values, e.g. 14px on property cards)
+  // with CSS token vars — rendered values capture overrides not present in token system.
+  // Filter: only simple single-value entries (reject shorthand "10px 10px 10px 2px").
+  const allRadiiMerged = new Set<string>([...radiusFromVars]);
+  for (const r of allBorderRadii as string[]) {
+    if (/^\d+(?:\.\d+)?px$/.test(r.trim())) {
+      allRadiiMerged.add(r.trim());
+    }
+  }
+  const sortedRadii = sortByPixels([...allRadiiMerged]);
+
+  // ── Semantic muted text color from CSS vars ──
+  // Find the best muted/secondary text color from CSS vars (e.g. --palette-icon-tertiary,
+  // --palette-text-secondary, --color-text-muted). For light sites: lum 0.15–0.65 range.
+  // Saturation filter (sat ≤ 30): muted text must be near-gray — excludes error reds, accents.
+  // Priority: icon/text vars first (sort by saturation ascending, then by key priority).
+  interface MutedCandidate { raw: string; sat: number; priority: number; }
+  const mutedTextCandidates: MutedCandidate[] = [];
+  for (const [k, v] of Object.entries(cssCustomProperties || {})) {
+    if (typeof v !== 'string') continue;
+    const vt = v.trim();
+    if (!/muted|tertiary|secondary|sub|icon|caption|quiet|helper|hint/i.test(k)) continue;
+    // Skip bg vars — those are backgrounds not text colors
+    if (/\bpb-bg\b|palette-bg|--bg-|color-bg|background/i.test(k)) continue;
+    // Parse hex or rgb — parseRgb only handles rgb() format
+    let rgb: [number, number, number] | null = parseRgb(vt);
+    if (!rgb) {
+      const hm = vt.match(/^#([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i);
+      if (hm) rgb = [parseInt(hm[1], 16), parseInt(hm[2], 16), parseInt(hm[3], 16)];
+    }
+    if (!rgb) continue;
+    const lum = luminance(rgb);
+    const sat = Math.max(...rgb) - Math.min(...rgb);
+    // Saturation guard: muted gray must have sat ≤ 30 (excludes #E00B41 sat=213, etc.)
+    if (sat > 30) continue;
+    // Luminance range: not too dark (same as primary), not too light (border territory)
+    const lumOk = !isDarkSite ? (lum >= 0.15 && lum <= 0.65) : (lum >= 0.40 && lum <= 0.85);
+    if (!lumOk) continue;
+    // Priority: prefer vars with "icon" or "text" scope over general secondary/tertiary
+    const priority = /\bicon\b|\btext\b/i.test(k) ? 0 : 1;
+    mutedTextCandidates.push({ raw: cleanColorValue(vt), sat, priority });
+  }
+  // Sort by priority (icon/text first), then by saturation ascending (most neutral)
+  mutedTextCandidates.sort((a, b) => a.priority - b.priority || a.sat - b.sat);
+  const resolvedMutedText = mutedTextCandidates[0]?.raw || (isDarkSite
+    ? (classified.text[2] || classified.text[1] || bodyColor)
+    : (classified.text[1] || classified.border[0] || bodyColor));
+
+  // ── Build tokens ──
+  const tokens: DesignTokens = {
+    meta: {
+      source: data.url,
+      domain,
+      extractedAt: data.timestamp,
+      tokenizedAt: new Date().toISOString(),
+    },
+    colors: {
+      background: {
+        primary: cleanColorValue(bodyBg),
+        secondary: cleanColorValue(classified.backgrounds[1] || bodyBg),
+        tertiary: cleanColorValue(classified.backgrounds[2] || classified.backgrounds[1] || bodyBg),
+      },
+      text: {
+        primary: cleanColorValue(bodyColor),
+        // Fix (audit-2026-05-28): ensure secondary is visually distinct from primary
+        secondary: cleanColorValue(pickDistinctTextSecondary(
+          dedupePerceptual(classified.text, 15),
+          bodyColor,
+          isDarkSite
+        )),
+        muted: resolvedMutedText,
+      },
+      accent: (() => {
+        // Phase 5.2.1 — Reject browser default colors (akiflow → #0000ee was being accepted as brand)
+        const BROWSER_DEFAULTS = new Set([
+          '#0000ee', '#0000EE', 'rgb(0, 0, 238)',          // default <a> link
+          '#551a8b', '#551A8B', 'rgb(85, 26, 139)',        // default :visited
+          '#ee0000', '#EE0000', 'rgb(238, 0, 0)',          // default :active
+        ]);
+        const isBrowserDefault = (c: string): boolean => {
+          if (!c) return false;
+          const norm = c.toLowerCase().replace(/\s+/g, '');
+          if (BROWSER_DEFAULTS.has(c)) return true;
+          // Check normalized forms
+          for (const d of BROWSER_DEFAULTS) {
+            if (d.toLowerCase().replace(/\s+/g, '') === norm) return true;
+          }
+          return false;
+        };
+
+        // Rank accents by DOM frequency (proxy = nb occurrences in allColors deduplicated list)
+        // + bonus +100 for CSS vars named --primary/--brand/--cta/--accent/--action
+        const freq = new Map<string, number>();
+        for (const c of allColors as string[]) {
+          if (isBrowserDefault(c)) continue;  // Phase 5.2.1 reject
+          const rgb = parseRgb(c);
+          if (!rgb) continue;
+          if (getSaturation(rgb) > 50 && luminance(rgb) > 0.1 && luminance(rgb) < 0.9) {
+            const norm = cleanColorValue(c);
+            freq.set(norm, (freq.get(norm) || 0) + 1);
+          }
+        }
+        // Bonus/penalty for semantically named CSS vars
+        for (const [k, v] of Object.entries(cssCustomProperties as Record<string, string>)) {
+          const vt = String(v).trim();
+          if (isBrowserDefault(vt)) continue;  // Phase 5.2.1 reject
+          const rgb = parseRgb(vt) ?? (() => {
+            const hm = vt.match(/^#([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i);
+            return hm ? [parseInt(hm[1], 16), parseInt(hm[2], 16), parseInt(hm[3], 16)] as [number, number, number] : null;
+          })();
+          if (!rgb) continue;
+          if (getSaturation(rgb) <= 50 || luminance(rgb) <= 0.1 || luminance(rgb) >= 0.9) continue;
+          const norm = cleanColorValue(vt);
+          if (/hover|active|pressed|focus(ed)?|disabled|error|danger|destructive|warning|alert|success|positive|confirm|info|notice|caution|inverse|dark-mode/i.test(k)) {
+            freq.set(norm, (freq.get(norm) || 0) - 200);
+          } else if (/primary|accent|brand|cta|action|button-bg/i.test(k)) {
+            freq.set(norm, (freq.get(norm) || 0) + 100);
+          }
+        }
+        const rankedEntries = [...freq.entries()].sort((a, b) => b[1] - a[1]);
+        const ranked = rankedEntries.map(([c]) => c);
+
+        // v2.6 Fix-1 — prefer RENDERED colors: accent.primary should actually appear on the page.
+        // Without this, a CSS-var-only value (e.g. miro `--tw-color-success-accent #00b473`, never
+        // painted) wins the +100 var-name bonus and becomes the brand color. We allow an unrendered
+        // color ONLY as a last resort and ONLY if it still carries a POSITIVE brand-var score (e.g.
+        // revolut's monochrome page whose violet lives in `--rui-color-accent #494fdf` + illustrations,
+        // never as a saturated element) — `success`/state vars now score negative and stay excluded.
+        const renderedSet = new Set((allColors as string[]).map(c => cleanColorValue(c)));
+        const rankedRendered = ranked.filter(c => renderedSet.has(c));
+        const rankedPositiveUnrendered = rankedEntries
+          .filter(([c, s]) => !renderedSet.has(c) && s > 0)
+          .map(([c]) => c);
+
+        // Highest-signal primary = background of the most prominent VIBRANT CTA button, read
+        // straight from the rendered DOM (beats frequency/var-name heuristics). Fixes notion
+        // (→ #455dd3 "Get Notion free") and miro (→ #fde050), which freq/var-bonus got wrong.
+        const ctaPrimary = (() => {
+          const btns = ((data as any).componentVariants?.buttons || []) as any[];
+          const cands = btns
+            .map(b => {
+              const bg = b?.styles?.backgroundColor || '';
+              const rgb = parseRgb(bg);
+              const area = (b?.rect?.width || 0) * (b?.rect?.height || 0);
+              return { bg, rgb, area };
+            })
+            .filter(c => !!c.rgb && !isBrowserDefault(c.bg)
+              && getSaturation(c.rgb as [number, number, number]) > 50
+              && luminance(c.rgb as [number, number, number]) > 0.1
+              && luminance(c.rgb as [number, number, number]) < 0.95);
+          cands.sort((a, b) => b.area - a.area);
+          return cands[0] ? cleanColorValue(cands[0].bg) : null;
+        })();
+
+        // classified.accents ⊂ allColors → always rendered; safe hard-blocked fallback.
+        const renderedFallback = classified.accents.find(a => !isBrowserDefault(a)) || null;
+        const ordered = [...(ctaPrimary ? [ctaPrimary] : []), ...rankedRendered, ...rankedPositiveUnrendered]
+          .filter((c, i, a) => a.indexOf(c) === i);
+        // Phase 5.2.1 — null if no rendered accent found (rather than mislabel an unrendered var).
+        return {
+          primary: ordered[0] || (renderedFallback ? cleanColorValue(renderedFallback) : null),
+          secondary: ordered[1] || ordered[0] || (renderedFallback ? cleanColorValue(renderedFallback) : null),
+        };
+      })(),
+      border: cleanColorValue(isDarkSite
+        ? (classified.border[0] || 'rgba(255,255,255,0.08)')
+        : (classified.border[0] || 'rgb(229,231,235)')),
+      shadow: isDarkSite ? 'rgba(0,0,0,0.3)' : 'rgba(0,0,0,0.1)',
+      semantic: (() => {
+        const result: Record<string, string> = {};
+        const PATTERNS: Array<[string, RegExp]> = [
+          ['error',   /\berror\b|\bdanger\b|\bdestructive\b/i],
+          ['success', /\bsuccess\b|\bpositive\b|\bconfirm\b/i],
+          ['warning', /\bwarn(ing)?\b|\bcaution\b/i],
+          ['info',    /\binfo(rmative)?\b|\bnotice\b/i],
+        ];
+        for (const [role, pattern] of PATTERNS) {
+          for (const [k, v] of Object.entries(cssCustomProperties as Record<string, string>)) {
+            if (!pattern.test(k)) continue;
+            const vt = String(v).trim();
+            const rgb = parseRgb(vt) ?? (() => {
+              const hm = vt.match(/^#([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i);
+              return hm ? [parseInt(hm[1], 16), parseInt(hm[2], 16), parseInt(hm[3], 16)] as [number, number, number] : null;
+            })();
+            if (rgb && getSaturation(rgb) > 40 && !result[role]) {
+              result[role] = cleanColorValue(vt);
+              break;
+            }
+          }
+        }
+        return Object.keys(result).length > 0 ? result : undefined;
+      })(),
+    },
+    typography: {
+      fontFamily: {
+        primary: uniqueFamilies[0] || 'system-ui',
+        secondary: uniqueFamilies[1] || uniqueFamilies[0] || 'system-ui',
+        mono: uniqueFamilies.find(f => /mono|code|consol/i.test(f)) || 'monospace',
+      },
+      fontSize: (() => {
+        // Smart scale: anchor on body base, distribute others across actual range
+        // FIX (audit-2026-05-28): guarantee monotonically increasing distinct values
+        // Avoid the degeneracy where lg=xl=2xl=3xl when only few sizes detected
+        const bodySize = elements.body?.styles?.fontSize || '16px';
+        const bodyPx = parsePixels(bodySize);
+
+        const unique = [...new Set(sortedSizes)].sort((a, b) => parsePixels(a) - parsePixels(b));
+        const displaySizes = unique.filter(s => parsePixels(s) >= bodyPx * 1.2);
+        const smallSizes = unique.filter(s => parsePixels(s) < bodyPx);
+
+        // Target scale (approximate ratios above body)
+        const targets = [
+          { name: 'lg', mul: 1.25 },
+          { name: 'xl', mul: 1.5 },
+          { name: '2xl', mul: 1.875 },
+          { name: '3xl', mul: 2.25 },
+          { name: '4xl', mul: 3.0 },
+        ];
+
+        // Pick distinct sizes by walking through display sizes,
+        // ensuring each subsequent token > previous (no degeneracy)
+        const pickedSizes: string[] = [];
+        let lastPx = bodyPx;
+        for (const t of targets) {
+          const targetPx = bodyPx * t.mul;
+          // First try: pick from displaySizes that are > lastPx and closest to targetPx
+          const candidates = displaySizes.filter(s => parsePixels(s) > lastPx);
+          let chosen: string;
+          if (candidates.length > 0) {
+            chosen = pickClosest(candidates, targetPx);
+          } else {
+            // Fallback: synthesize the target value, guarantee strictly > lastPx
+            const synthPx = Math.max(Math.round(targetPx), Math.round(lastPx + 2));
+            chosen = synthPx + 'px';
+          }
+          pickedSizes.push(chosen);
+          lastPx = parsePixels(chosen);
+        }
+
+        const xs = smallSizes[0] || pickClosest(sortedSizes, 12);
+        const sm = smallSizes.length > 1
+          ? smallSizes[smallSizes.length - 1]
+          : (parsePixels(xs) < bodyPx - 1 ? Math.round((parsePixels(xs) + bodyPx) / 2) + 'px' : pickClosest(sortedSizes, 14));
+
+        return {
+          xs,
+          sm,
+          base: bodySize,
+          lg: pickedSizes[0],
+          xl: pickedSizes[1],
+          '2xl': pickedSizes[2],
+          '3xl': pickedSizes[3],
+          '4xl': pickedSizes[4],
+        };
+      })(),
+      fontWeight: (() => {
+        // Phase 5.1.2 — Collect from BOTH elements AND componentVariants (was only KEY_SELECTORS before)
+        // Previously: 9/30 sites showed false "Don't use weight 700" because variants weren't sampled.
+        const weights = new Set<number>();
+        for (const el of Object.values(elements)) {
+          const fw = (el as any)?.styles?.fontWeight;
+          if (fw && /^\d+$/.test(String(fw).trim())) weights.add(Number(fw));
+        }
+        // Sample componentVariants too — heading variants often have weights NOT visible in KEY_SELECTORS
+        const variants = (data as any).componentVariants || {};
+        for (const group of Object.values(variants)) {
+          if (!Array.isArray(group)) continue;
+          for (const v of group) {
+            const fw = v?.styles?.fontWeight;
+            if (fw && /^\d+$/.test(String(fw).trim())) weights.add(Number(fw));
+          }
+        }
+        const sorted = [...weights].sort((a, b) => a - b);
+        return {
+          normal:   String(sorted.find(w => w <= 400) ?? 400),
+          medium:   String(sorted.find(w => w > 400 && w <= 500) ?? sorted.find(w => w > 400) ?? 500),
+          semibold: String(sorted.find(w => w >= 550 && w <= 650) ?? sorted.find(w => w > 500) ?? 600),
+          bold:     String(sorted.find(w => w >= 700) ?? sorted[sorted.length - 1] ?? 700),
+        };
+      })(),
+      lineHeight: {
+        tight: normalizeLineHeight(elements.heading?.styles?.lineHeight, elements.heading?.styles?.fontSize) || '1.25',
+        normal: normalizeLineHeight(elements.body?.styles?.lineHeight, elements.body?.styles?.fontSize) || '1.5',
+        relaxed: normalizeLineHeight(elements.nav?.styles?.lineHeight, elements.nav?.styles?.fontSize) || '1.75',
+      },
+      letterSpacing: (() => {
+        // Extract real letter-spacing from heading/subheading variants — never hardcode
+        const h1Variants: any[] = (data as any).componentVariants?.headingH1 || [];
+        const h2Variants: any[] = (data as any).componentVariants?.headingH2 || [];
+        const toEm = (ls: string, fsStr?: string): string => {
+          if (!ls || ls === 'normal') return '0em';
+          if (ls.includes('em')) return ls;
+          const lsPx = parsePixels(ls);
+          if (lsPx === 0) return '0em';
+          const fsPx = fsStr ? parsePixels(fsStr) : 16;
+          return `${(lsPx / (fsPx || 16)).toFixed(3)}em`;
+        };
+        const rawValues: string[] = [];
+        for (const v of [...h1Variants, ...h2Variants]) {
+          const ls = v?.styles?.letterSpacing;
+          if (ls && ls !== '0px' && ls !== 'normal') rawValues.push(ls);
+        }
+        const headingLs = elements.heading?.styles?.letterSpacing;
+        if (headingLs && headingLs !== '0px' && headingLs !== 'normal') rawValues.push(headingLs);
+        const bodyLs = elements.body?.styles?.letterSpacing;
+        const tight = rawValues[0]
+          ? toEm(rawValues[0], elements.heading?.styles?.fontSize)
+          : '-0.025em';
+        const wide = rawValues.length > 1
+          ? toEm(rawValues[rawValues.length - 1], elements.heading?.styles?.fontSize)
+          : (elements.button?.styles?.letterSpacing !== 'normal' && elements.button?.styles?.letterSpacing !== '0px'
+              ? toEm(elements.button?.styles?.letterSpacing || '0', elements.button?.styles?.fontSize)
+              : '0.025em');
+        return {
+          tight,
+          normal: toEm(bodyLs || '0', elements.body?.styles?.fontSize),
+          wide,
+        };
+      })(),
+    },
+    spacing: (() => {
+      const scaleTargets = [4, 8, 12, 16, 24, 32, 48, 64];
+      const scaleValues = pickDistinctScale(sortedSpacing, scaleTargets);
+      return {
+        xxs: '2px',
+        xs: scaleValues[0],
+        sm: scaleValues[1],
+        md: scaleValues[2],
+        base: scaleValues[3],
+        lg: scaleValues[4],
+        xl: scaleValues[5],
+        '2xl': scaleValues[6],
+        '3xl': scaleValues[7],
+      };
+    })(),
+    borderRadius: {
+      none: '0px',
+      xs: pickClosest(sortedRadii, 4),
+      sm: pickClosest(sortedRadii, 8),
+      md: pickClosest(sortedRadii, 14),  // card radius — rendered value (14px on property cards)
+      lg: pickClosest(sortedRadii, 20),  // large rounded elements (pill buttons, search segments)
+      xl: pickClosest(sortedRadii, 32),  // category strips, large containers
+      full: sortedRadii.find(r => parsePixels(r) >= 999) || '9999px',
+    },
+    shadows: Object.fromEntries(
+      allShadows.slice(0, 5).map((s: string, i: number) => [`shadow-${i + 1}`, s])
+    ),
+    transitions: Object.fromEntries(
+      allTransitions.slice(0, 5).map((t: string, i: number) => [`transition-${i + 1}`, t])
+    ),
+    layout: {
+      maxWidth: resolveMaxWidth(elements, cssCustomProperties || {}),
+      headerHeight: elements.header ? `${elements.header.rect.height}px` : '64px',
+      sidebarWidth: elements.sidebar ? `${elements.sidebar.rect.width}px` : '0px',
+      gap: elements.main?.styles?.gap || '16px',
+      containerPadding: elements.main?.styles?.padding || '24px',
+    },
+    cssCustomProperties: cssCustomProperties || {},
+  };
+
+  // Phase 5.2.2 — Contrast sanity check. If text.primary and bg.primary have <3:1 ratio,
+  // one of them is wrong (Framer/Shopify black-on-black, MongoDB/Notion near-black on dark).
+  // Strategy: trust bg from body, recalculate text from h1 element if conflict.
+  const _parseColorRgb = (s: string): [number, number, number] | null => {
+    const rgb = parseRgb(s);
+    if (rgb) return rgb;
+    const hm = s.match(/^#([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i);
+    return hm ? [parseInt(hm[1], 16), parseInt(hm[2], 16), parseInt(hm[3], 16)] : null;
+  };
+  const _wcagContrast = (rgb1: [number, number, number], rgb2: [number, number, number]): number => {
+    const lum = (rgb: [number, number, number]): number => {
+      const [r, g, b] = rgb.map(v => {
+        const n = v / 255;
+        return n <= 0.03928 ? n / 12.92 : Math.pow((n + 0.055) / 1.055, 2.4);
+      });
+      return 0.2126 * r + 0.7152 * g + 0.0722 * b;
+    };
+    const l1 = lum(rgb1);
+    const l2 = lum(rgb2);
+    const lighter = Math.max(l1, l2);
+    const darker = Math.min(l1, l2);
+    return (lighter + 0.05) / (darker + 0.05);
+  };
+  const _textRgb = _parseColorRgb(tokens.colors.text.primary || '');
+  const _bgRgb = _parseColorRgb(tokens.colors.background.primary || '');
+  if (_textRgb && _bgRgb) {
+    const contrast = _wcagContrast(_textRgb, _bgRgb);
+    if (contrast < 3.0) {
+      console.warn(`  ⚠️  Phase 5.2.2: text/bg contrast ${contrast.toFixed(2)}:1 < 3 — fallback to h1 color`);
+      const h1Color = elements.h1?.styles?.color || elements.heading?.styles?.color;
+      if (h1Color) {
+        const newRgb = _parseColorRgb(h1Color);
+        if (newRgb && _wcagContrast(newRgb, _bgRgb) >= 3.0) {
+          tokens.colors.text.primary = cleanColorValue(h1Color);
+          console.warn(`     → text.primary corrected to ${tokens.colors.text.primary} (h1 color)`);
+        }
+      }
+    }
+  }
+
+  const outputPath = join(baseDir, 'tokens.json');
+  await writeFile(outputPath, JSON.stringify(tokens, null, 2));
+
+  console.log(`✅ Tokens saved to ${outputPath}`);
+  console.log(`   ${Object.keys(cssCustomProperties || {}).length} CSS custom properties preserved`);
+  console.log(`   Colors: ${allColors.length} found → ${Object.keys(tokens.colors).length} categories`);
+  console.log(`   Fonts: ${uniqueFamilies.length} families, ${sortedSizes.length} sizes`);
+  console.log(`   Spacing: ${sortedSpacing.length} values, Radii: ${sortedRadii.length} values`);
+}
+
+// ── CLI ──
+const domain = process.argv[2];
+if (!domain) {
+  console.error('Usage: npm run tokenize -- <domain>');
+  process.exit(1);
+}
+
+tokenize(domain).catch(err => {
+  console.error('Error:', err);
+  process.exit(1);
+});