prism-design 2.13.0

package/scripts/mass-extract.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+# Mass extraction orchestrator — 71 GD brands en batches de 3 parallèles
+# Tier 1 (easy) → Tier 2 (medium) → Tier 3 (risky)
+# Cumulative timeout: 240s par extract (skip après timeout)
+# Output: logs/mass-<TS>/{success,failed,timing}.txt
+
+set -u
+cd /home/paul/clone-architect
+TS=$(date +%Y%m%d-%H%M)
+LOGDIR="logs/mass-$TS"
+mkdir -p "$LOGDIR"
+SUCCESS="$LOGDIR/success.txt"
+FAILED="$LOGDIR/failed.txt"
+TIMING="$LOGDIR/timing.txt"
+touch "$SUCCESS" "$FAILED" "$TIMING"
+
+# === Tier 1 — Easy wins (33 brands) ===
+TIER1=(claude.com clay.com cohere.com composio.dev expo.dev hashicorp.com mastercard.com replicate.com sanity.io sentry.io superhuman.com voltagent.dev warp.dev wise.com x.ai zapier.com airtable.com elevenlabs.io pinterest.com runwayml.com cal.com clickhouse.com intercom.com lovable.dev mintlify.com mistral.ai opencode.ai posthog.com resend.com supabase.com together.ai raycast.com linear.app)
+
+# === Tier 2 — Medium (24 brands) ===
+TIER2=(bmw.com bmwm.com bugatti.com ferrari.com hp.com lamborghini.com meta.com playstation.com renault.com spacex.com tesla.com uber.com wired.com airbnb.com apple.com cursor.com framer.com ibm.com miro.com mongodb.com nvidia.com ollama.com shopify.com spotify.com)
+
+# === Tier 3 — Risky (14 brands, anti-bot) ===
+TIER3=(binance.com coinbase.com kraken.com nike.com notion.so revolut.com starbucks.com stripe.com theverge.com vercel.com vodafone.com webflow.com minimax.io figma.com)
+
+extract_one() {
+  local domain=$1
+  local start=$(date +%s)
+  # bmwm.com special case (BMW M doesn't exist as TLD, use bmw-m.com or skip)
+  local url="https://$domain"
+  if [ "$domain" = "bmwm.com" ]; then
+    url="https://www.bmw-m.com"
+  fi
+  if timeout 240s npx tsx scripts/extract.ts "$url" > "$LOGDIR/$domain.log" 2>&1; then
+    local elapsed=$(( $(date +%s) - start ))
+    echo "$domain" >> "$SUCCESS"
+    echo "$domain $elapsed" >> "$TIMING"
+    echo "  ✅ $domain (${elapsed}s)"
+  else
+    local elapsed=$(( $(date +%s) - start ))
+    echo "$domain" >> "$FAILED"
+    echo "$domain ${elapsed}s FAIL" >> "$TIMING"
+    echo "  ❌ $domain (${elapsed}s)"
+  fi
+}
+
+run_batch_of_3() {
+  local d1=${1:-}; local d2=${2:-}; local d3=${3:-}
+  [ -n "$d1" ] && extract_one "$d1" &
+  [ -n "$d2" ] && extract_one "$d2" &
+  [ -n "$d3" ] && extract_one "$d3" &
+  wait
+}
+
+run_tier() {
+  local tier_name=$1
+  shift
+  local tier=("$@")
+  echo "═══════════════════════════════════════════════════"
+  echo "🚀 $tier_name — ${#tier[@]} brands"
+  echo "═══════════════════════════════════════════════════"
+  local i=0
+  while [ $i -lt ${#tier[@]} ]; do
+    local d1=${tier[$i]:-}
+    local d2=${tier[$((i+1))]:-}
+    local d3=${tier[$((i+2))]:-}
+    run_batch_of_3 "$d1" "$d2" "$d3"
+    i=$((i+3))
+    # Free RAM between batches
+    sync && echo 3 | sudo tee /proc/sys/vm/drop_caches > /dev/null 2>&1 || true
+  done
+}
+
+echo "📁 Logs: $LOGDIR"
+echo ""
+
+run_tier "TIER 1 — Easy" "${TIER1[@]}"
+run_tier "TIER 2 — Medium" "${TIER2[@]}"
+run_tier "TIER 3 — Risky" "${TIER3[@]}"
+
+echo ""
+echo "═══════════════════════════════════════════════════"
+echo "📊 RÉSULTATS"
+echo "═══════════════════════════════════════════════════"
+echo "  Success : $(wc -l < $SUCCESS) brands"
+echo "  Failed  : $(wc -l < $FAILED) brands"
+echo ""
+echo "Failed brands:"
+cat "$FAILED" 2>/dev/null | sed 's/^/  ❌ /'
+echo ""
+echo "Logs in: $LOGDIR"

package/scripts/post-process-all.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Post-process all extractions: tokenize + generate-design-md + generate-showcase
+# Adds new brands to catalog and runs enrichment
+
+set -u
+cd /home/paul/clone-architect
+
+LOGDIR="logs/postproc-$(date +%Y%m%d-%H%M)"
+mkdir -p "$LOGDIR"
+
+echo "🔧 Post-processing all extractions..."
+
+SUCCESS=0
+SKIPPED=0
+FAILED=0
+
+for dir in extractions/*/; do
+  domain=$(basename "$dir")
+  if [ -f "$dir/raw-css.json" ]; then
+    if npx tsx scripts/tokenize.ts "$domain" > "$LOGDIR/$domain.tok.log" 2>&1; then
+      if npx tsx scripts/generate-design-md.ts "$domain" > "$LOGDIR/$domain.dmd.log" 2>&1; then
+        SUCCESS=$((SUCCESS+1))
+        echo "  ✅ $domain"
+      else
+        FAILED=$((FAILED+1))
+        echo "  ❌ $domain (design-md failed)"
+      fi
+    else
+      FAILED=$((FAILED+1))
+      echo "  ❌ $domain (tokenize failed)"
+    fi
+  else
+    SKIPPED=$((SKIPPED+1))
+    echo "  ⏭  $domain (no raw-css.json)"
+  fi
+done
+
+echo ""
+echo "📊 Tokenize + DESIGN.md: $SUCCESS OK · $FAILED FAIL · $SKIPPED SKIP"
+echo ""
+echo "🎨 Generating showcases..."
+
+SHOW_OK=0
+SHOW_FAIL=0
+for dir in extractions/*/; do
+  domain=$(basename "$dir")
+  if [ -f "$dir/DESIGN.md" ]; then
+    if npx tsx scripts/generate-showcase.ts "$domain" > "$LOGDIR/$domain.show.log" 2>&1; then
+      SHOW_OK=$((SHOW_OK+1))
+    else
+      SHOW_FAIL=$((SHOW_FAIL+1))
+    fi
+  fi
+done
+echo "📊 Showcases: $SHOW_OK OK · $SHOW_FAIL FAIL"

package/scripts/regen-catalog.ts

@@ -0,0 +1,203 @@
+/**
+ * Prism — Batch DESIGN.md Regeneration
+ *
+ * Regenerates DESIGN.md for all extractions that have both raw-css.json + tokens.json.
+ * Defensive: skips extractions with missing files (never process.exit on partial failure).
+ * Reports a final summary with scores.
+ *
+ * Usage: npx tsx scripts/regen-catalog.ts [--dry-run] [--domain <domain>]
+ *   --dry-run    Show what would be regenerated without writing files
+ *   --domain     Regenerate a single domain only
+ */
+
+import { readdir, readFile, stat } from 'fs/promises';
+import { existsSync } from 'fs';
+import { join } from 'path';
+import { spawnSync } from 'child_process';
+
+const ROOT = process.cwd();
+const EXTRACTIONS_DIR = join(ROOT, 'extractions');
+const GENERATE_SCRIPT = join(ROOT, 'scripts', 'generate-design-md.ts');
+
+const isDryRun = process.argv.includes('--dry-run');
+const domainFilter = (() => {
+  const idx = process.argv.indexOf('--domain');
+  return idx >= 0 ? process.argv[idx + 1] : null;
+})();
+
+interface RegenResult {
+  domain: string;
+  status: 'ok' | 'skipped' | 'error';
+  reason?: string;
+  completeness?: number;
+  lines?: number;
+}
+
+async function getCompleteness(designMdPath: string): Promise<number | null> {
+  try {
+    const content = await readFile(designMdPath, 'utf-8');
+    const m = content.match(/^completeness:\s*(\d+)/m);
+    return m ? parseInt(m[1]) : null;
+  } catch {
+    return null;
+  }
+}
+
+async function getLineCount(designMdPath: string): Promise<number> {
+  try {
+    const content = await readFile(designMdPath, 'utf-8');
+    return content.split('\n').length;
+  } catch {
+    return 0;
+  }
+}
+
+async function main() {
+  console.log(`\n🔄 Prism — Batch DESIGN.md Regeneration`);
+  if (isDryRun) console.log(`   DRY RUN — no files will be written`);
+  if (domainFilter) console.log(`   Filter: ${domainFilter} only`);
+  console.log('');
+
+  // Discover all extraction directories
+  let allDomains: string[] = [];
+  try {
+    const entries = await readdir(EXTRACTIONS_DIR);
+    for (const entry of entries) {
+      const fullPath = join(EXTRACTIONS_DIR, entry);
+      try {
+        const s = await stat(fullPath);
+        if (s.isDirectory()) allDomains.push(entry);
+      } catch { /* skip */ }
+    }
+  } catch (err) {
+    console.error('Cannot read extractions directory:', err);
+    process.exit(1);
+  }
+
+  if (domainFilter) {
+    allDomains = allDomains.filter(d => d === domainFilter);
+    if (allDomains.length === 0) {
+      console.error(`Domain "${domainFilter}" not found in extractions/`);
+      process.exit(1);
+    }
+  }
+
+  allDomains.sort();
+  console.log(`Found ${allDomains.length} extraction(s) to process...\n`);
+
+  const results: RegenResult[] = [];
+  let okCount = 0;
+  let skipCount = 0;
+  let errorCount = 0;
+
+  for (const domain of allDomains) {
+    const extractionDir = join(EXTRACTIONS_DIR, domain);
+    const rawCssPath = join(extractionDir, 'raw-css.json');
+    const tokensPath = join(extractionDir, 'tokens.json');
+    const designMdPath = join(extractionDir, 'DESIGN.md');
+
+    // Skip if missing required input files
+    if (!existsSync(rawCssPath)) {
+      results.push({ domain, status: 'skipped', reason: 'no raw-css.json' });
+      skipCount++;
+      console.log(`  ⏭️  ${domain} — skipped (no raw-css.json)`);
+      continue;
+    }
+    if (!existsSync(tokensPath)) {
+      results.push({ domain, status: 'skipped', reason: 'no tokens.json' });
+      skipCount++;
+      console.log(`  ⏭️  ${domain} — skipped (no tokens.json)`);
+      continue;
+    }
+
+    if (isDryRun) {
+      console.log(`  🔲 ${domain} — would regenerate`);
+      results.push({ domain, status: 'ok', reason: 'dry-run' });
+      okCount++;
+      continue;
+    }
+
+    // Resolve tsx binary
+    const localTsx = join(ROOT, 'node_modules', '.bin', 'tsx');
+    const tsxCmd = existsSync(localTsx) ? localTsx : 'npx';
+    const tsxArgs = existsSync(localTsx)
+      ? [GENERATE_SCRIPT, domain]
+      : ['tsx', GENERATE_SCRIPT, domain];
+
+    process.stdout.write(`  ⚙️  ${domain}... `);
+
+    const result = spawnSync(tsxCmd, tsxArgs, {
+      cwd: ROOT,
+      encoding: 'utf-8',
+      timeout: 30000, // 30s max per domain
+    });
+
+    if (result.error || result.status !== 0) {
+      const errMsg = result.stderr?.split('\n')[0] || result.error?.message || 'unknown error';
+      console.log(`❌ ${errMsg.slice(0, 80)}`);
+      results.push({ domain, status: 'error', reason: errMsg });
+      errorCount++;
+      continue;
+    }
+
+    // Read back completeness score
+    const completeness = await getCompleteness(designMdPath);
+    const lines = await getLineCount(designMdPath);
+
+    const scoreDisplay = completeness !== null ? `${completeness}/100` : 'no score';
+    console.log(`✅ ${lines}L | ${scoreDisplay}`);
+    results.push({ domain, status: 'ok', completeness: completeness ?? undefined, lines });
+    okCount++;
+  }
+
+  // ── Summary ──
+  console.log('\n' + '═'.repeat(60));
+  console.log(`📊 Regeneration Summary`);
+  console.log(`   ✅ OK: ${okCount} | ⏭️  Skipped: ${skipCount} | ❌ Errors: ${errorCount}`);
+
+  if (!isDryRun) {
+    const withScores = results.filter(r => r.completeness !== undefined);
+    if (withScores.length > 0) {
+      const avgScore = Math.round(withScores.reduce((sum, r) => sum + (r.completeness!), 0) / withScores.length);
+      const maxScore = Math.max(...withScores.map(r => r.completeness!));
+      const minScore = Math.min(...withScores.map(r => r.completeness!));
+      console.log(`   Completeness: avg ${avgScore}/100 | best ${maxScore} | worst ${minScore}`);
+
+      // Top 5 and bottom 5
+      const sorted = [...withScores].sort((a, b) => b.completeness! - a.completeness!);
+      console.log(`\n   Top 5:`);
+      for (const r of sorted.slice(0, 5)) {
+        console.log(`     ${r.domain}: ${r.completeness}/100`);
+      }
+      if (sorted.length > 5) {
+        console.log(`\n   Needs improvement:`);
+        for (const r of sorted.slice(-5).reverse()) {
+          console.log(`     ${r.domain}: ${r.completeness}/100`);
+        }
+      }
+    }
+
+    // Run catalog enrichment to update catalog/index.json with new scores
+    console.log('\n🔧 Updating catalog/index.json with new scores...');
+    const localTsx3 = join(ROOT, 'node_modules', '.bin', 'tsx');
+    const tsxCmd3 = existsSync(localTsx3) ? localTsx3 : 'npx';
+    const enrichScript = join(ROOT, 'scripts', 'enrich-catalog.ts');
+    if (existsSync(enrichScript)) {
+      const enrichArgs = existsSync(localTsx3)
+        ? [enrichScript]
+        : ['tsx', enrichScript];
+      const enrichResult = spawnSync(tsxCmd3, enrichArgs, { stdio: 'inherit', cwd: ROOT });
+      if (enrichResult.status !== 0) {
+        console.warn('⚠️  catalog enrichment failed — run manually: npx tsx scripts/enrich-catalog.ts');
+      }
+    }
+  }
+
+  console.log('');
+  process.exit(errorCount > 0 ? 1 : 0);
+}
+
+main().catch(err => {
+  console.error('Fatal error in regen-catalog:', err);
+  process.exit(1);
+});

package/scripts/shared/cache.ts

@@ -0,0 +1,149 @@
+/**
+ * shared/cache.ts — Phase 2.3 re-extraction cache
+ *
+ * Skips Playwright extraction if the URL content hasn't materially changed since last run.
+ * Fingerprint = hash(URL + HTTP headers ETag/last-modified/content-length + algorithm version).
+ *
+ * Cache hit: ~2s (HEAD request only) vs full extraction ~140s = 99% time saved on re-runs.
+ *
+ * Safety:
+ * - Atomic writes (temp file + rename) prevent corruption on partial writes
+ * - Algorithm version baked into fingerprint = automatic bust when extractors change
+ * - Force flag bypasses cache for explicit re-extractions
+ * - TTL prevents stale fingerprints from blocking forever
+ */
+
+import { createHash } from 'crypto';
+import { existsSync, readFileSync, writeFileSync, renameSync, statSync } from 'fs';
+import { join, dirname } from 'path';
+import { mkdir } from 'fs/promises';
+
+/** Algorithm version — bump when extraction logic materially changes. Bust all caches. */
+export const CACHE_ALGORITHM_VERSION = '2026.05.26';
+
+export interface CacheCheckResult {
+  skip: boolean;
+  reason: string;
+  fingerprint?: string;
+}
+
+/**
+ * Check if extraction should be skipped for this URL.
+ * Performs a HEAD request to detect URL content changes via standard HTTP headers.
+ * Returns {skip:true} when fingerprints match AND TTL not exceeded.
+ *
+ * @param url - Target URL to potentially re-extract
+ * @param domain - Resolved domain (extractions/<domain>/ dir)
+ * @param opts - Options
+ *   - force: bypass cache entirely (always extract)
+ *   - ttlDays: max age before re-extraction even on match (default 7)
+ *   - cacheDir: extractions root (default 'extractions')
+ */
+export async function shouldSkipExtraction(
+  url: string,
+  domain: string,
+  opts: { force?: boolean; ttlDays?: number; cacheDir?: string } = {}
+): Promise<CacheCheckResult> {
+  if (opts.force) return { skip: false, reason: 'force flag set' };
+
+  const cacheDir = opts.cacheDir || 'extractions';
+  const ttlDays = opts.ttlDays ?? 7;
+  const fpPath = join(cacheDir, domain, '.extraction-fingerprint');
+  const rawCssPath = join(cacheDir, domain, 'raw-css.json');
+
+  // No existing extraction → must run, but still compute fingerprint for post-extraction write
+  if (!existsSync(rawCssPath) || !existsSync(fpPath)) {
+    let fp: string | undefined;
+    try {
+      fp = await computeUrlFingerprint(url);
+    } catch {
+      // HEAD fail → no fingerprint (cache cannot be populated this run)
+    }
+    return { skip: false, reason: 'no prior extraction or fingerprint', fingerprint: fp };
+  }
+
+  // TTL check
+  try {
+    const stat = statSync(fpPath);
+    const ageMs = Date.now() - stat.mtimeMs;
+    const ttlMs = ttlDays * 86400 * 1000;
+    if (ageMs > ttlMs) {
+      return { skip: false, reason: `fingerprint older than ${ttlDays}d TTL` };
+    }
+  } catch {
+    return { skip: false, reason: 'cannot stat fingerprint file' };
+  }
+
+  // Compute current fingerprint from URL HEAD
+  let newFp: string;
+  try {
+    newFp = await computeUrlFingerprint(url);
+  } catch (e) {
+    return { skip: false, reason: `HEAD request failed: ${(e as Error).message}` };
+  }
+
+  // Compare with stored fingerprint
+  try {
+    const storedFp = readFileSync(fpPath, 'utf-8').trim();
+    if (storedFp === newFp) {
+      return { skip: true, reason: 'fingerprint match (unchanged)', fingerprint: newFp };
+    }
+    return { skip: false, reason: 'fingerprint differs (content or version changed)', fingerprint: newFp };
+  } catch {
+    return { skip: false, reason: 'cannot read stored fingerprint' };
+  }
+}
+
+/**
+ * Write fingerprint atomically (temp file + rename) so partial writes never corrupt cache.
+ */
+export async function writeFingerprint(
+  domain: string,
+  fingerprint: string,
+  opts: { cacheDir?: string } = {}
+): Promise<void> {
+  const cacheDir = opts.cacheDir || 'extractions';
+  const finalPath = join(cacheDir, domain, '.extraction-fingerprint');
+  const tmpPath = `${finalPath}.${process.pid}.${Date.now()}.tmp`;
+
+  await mkdir(dirname(finalPath), { recursive: true });
+  writeFileSync(tmpPath, fingerprint, 'utf-8');
+  renameSync(tmpPath, finalPath);  // atomic on POSIX filesystems
+}
+
+/**
+ * Compute a fingerprint from URL + HTTP headers + algorithm version.
+ * Strategy:
+ *   1. HEAD request → get ETag / Last-Modified / Content-Length / Content-Type
+ *   2. Hash(URL + headers + ALGORITHM_VERSION)
+ *   3. Returns hex string (40 chars sha1)
+ *
+ * Why include algorithm version: if extractors evolve (new fields, fixed bugs), cache
+ * must invalidate even when content is identical.
+ */
+async function computeUrlFingerprint(url: string): Promise<string> {
+  let etag = '';
+  let lastMod = '';
+  let contentLen = '';
+  let contentType = '';
+
+  try {
+    const response = await fetch(url, {
+      method: 'HEAD',
+      redirect: 'follow',
+      signal: AbortSignal.timeout(8000),
+    });
+    if (response.ok || response.status === 405) {
+      // Some sites reject HEAD (405) but headers may still be informative
+      etag = response.headers.get('etag') || '';
+      lastMod = response.headers.get('last-modified') || '';
+      contentLen = response.headers.get('content-length') || '';
+      contentType = response.headers.get('content-type') || '';
+    }
+  } catch {
+    // HEAD failed (CORS, timeout, network). Fall through with empty headers.
+  }
+
+  const material = [url, etag, lastMod, contentLen, contentType, CACHE_ALGORITHM_VERSION].join('|');
+  return createHash('sha1').update(material).digest('hex');
+}