similarbuild 0.1.0

package/templates/skills/sb-extract-assets/scripts/extract-assets.mjs

@@ -0,0 +1,484 @@
+#!/usr/bin/env node
+// extract-assets.mjs — Download images from sb-inspect-live's imgUrls list,
+// strip identifying metadata (EXIF/XMP/IPTC), rename via sha256 content-hash,
+// dedupe against assets already on disk, and emit assets-map.json.
+//
+// Strip is the *default* sharp behaviour: we re-encode without calling
+// .withMetadata(). .rotate() is called BEFORE the strip so EXIF orientation
+// is baked into pixels (otherwise an upright phone photo looks sideways once
+// the EXIF block is gone). ICC profile is preserved for color fidelity.
+//
+// SVGs don't have EXIF but carry editor signatures (`<metadata>`, comments,
+// inkscape:/sodipodi: namespaces) — those are stripped via regex. For
+// target=wp the sanitized markup is returned inline (WP blocks raw SVG
+// uploads by default). For target=shopify SVGs are saved as files.
+//
+// Output: JSON to stdout AND assets-map.json in --output-dir. Logs to stderr.
+
+import { parseArgs } from 'node:util'
+import { mkdir, writeFile, readFile, readdir } from 'node:fs/promises'
+import { existsSync } from 'node:fs'
+import { join, resolve, basename, extname } from 'node:path'
+import { createHash } from 'node:crypto'
+
+// sharp is imported lazily inside main() so --help and arg validation
+// work without the dep installed.
+
+const HELP = `
+extract-assets.mjs — Download + sanitize images for SimilarBuild visual cloning.
+
+One of:
+  --inspection-path <file>   Path to inspection.json (script reads imgUrls).
+  --img-urls <json>          Inline JSON array [{url, context, alt}].
+
+Required:
+  --output-dir <dir>         Directory for sanitized assets + assets-map.json.
+
+Optional:
+  --target <wp|shopify>      Default 'wp'. Affects SVG handling (inline vs file).
+  --existing-assets-dir <d>  Extra dir of already-extracted assets to dedupe against.
+  --large-warn-mb <n>        Warn (stderr) if asset > N MB. Default 10.
+  --timeout <ms>             Per-request fetch timeout. Default 30000.
+  --help                     Show this message.
+
+Exit codes: 0=ok, 1=script error, 2=invalid args.
+`
+
+function fail(msg, code = 2) {
+  process.stderr.write(`[sb-extract-assets] ${msg}\n`)
+  process.exit(code)
+}
+
+function log(msg) {
+  process.stderr.write(`[sb-extract-assets] ${msg}\n`)
+}
+
+const { values } = parseArgs({
+  options: {
+    'inspection-path': { type: 'string' },
+    'img-urls': { type: 'string' },
+    'output-dir': { type: 'string' },
+    target: { type: 'string', default: 'wp' },
+    'existing-assets-dir': { type: 'string' },
+    'large-warn-mb': { type: 'string', default: '10' },
+    timeout: { type: 'string', default: '30000' },
+    help: { type: 'boolean', default: false },
+  },
+  strict: false,
+})
+
+if (values.help) {
+  process.stdout.write(HELP)
+  process.exit(0)
+}
+
+if (!values['output-dir']) fail('missing --output-dir')
+if (!values['inspection-path'] && !values['img-urls']) {
+  fail('missing input: pass --inspection-path or --img-urls')
+}
+if (values['inspection-path'] && values['img-urls']) {
+  fail('pass exactly one of --inspection-path or --img-urls, not both')
+}
+if (!['wp', 'shopify'].includes(values.target)) {
+  fail(`invalid --target '${values.target}' (must be 'wp' or 'shopify')`)
+}
+
+const OUTPUT_DIR = resolve(values['output-dir'])
+const TARGET = values.target
+const EXISTING_DIR = values['existing-assets-dir'] ? resolve(values['existing-assets-dir']) : null
+const LARGE_WARN_BYTES = parseInt(values['large-warn-mb'], 10) * 1024 * 1024
+const TIMEOUT = parseInt(values.timeout, 10)
+
+if (!Number.isFinite(LARGE_WARN_BYTES)) fail('--large-warn-mb must be numeric')
+if (!Number.isFinite(TIMEOUT)) fail('--timeout must be numeric')
+
+// Validate input source up front (before sharp import) so input errors exit
+// with code 2 (invalid args) instead of code 1 (script error from missing dep).
+let INPUT_URLS = null
+if (values['img-urls']) {
+  try {
+    const arr = JSON.parse(values['img-urls'])
+    if (!Array.isArray(arr)) fail('--img-urls must be a JSON array')
+    INPUT_URLS = arr
+  } catch (err) {
+    if (err?.message?.startsWith('--img-urls')) throw err
+    fail(`--img-urls is not valid JSON: ${err.message}`)
+  }
+} else {
+  const p = resolve(values['inspection-path'])
+  if (!existsSync(p)) fail(`inspection file not found: ${p}`)
+  // Defer parse to main() — file exists, that's enough for arg validation.
+}
+
+// MIME → extension. Authoritative: drives the saved filename's extension,
+// regardless of what the URL or Content-Disposition claims. Prevents the
+// classic ".jpg?v=2 actually a PNG" mismatch.
+const MIME_EXT = {
+  'image/jpeg': 'jpg',
+  'image/jpg': 'jpg',
+  'image/png': 'png',
+  'image/webp': 'webp',
+  'image/avif': 'avif',
+  'image/gif': 'gif',
+  'image/svg+xml': 'svg',
+  'image/x-icon': 'ico',
+  'image/vnd.microsoft.icon': 'ico',
+  'image/heic': 'heic',
+  'image/heif': 'heif',
+  'image/bmp': 'bmp',
+  'image/tiff': 'tiff',
+}
+
+// Magic-byte sniff fallback when Content-Type lies or is generic
+// (octet-stream, text/plain). Covers the formats sharp can decode.
+function sniffMime(buf) {
+  if (buf.length >= 12) {
+    if (buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) return 'image/jpeg'
+    if (
+      buf[0] === 0x89 &&
+      buf[1] === 0x50 &&
+      buf[2] === 0x4e &&
+      buf[3] === 0x47
+    ) return 'image/png'
+    if (buf[0] === 0x47 && buf[1] === 0x49 && buf[2] === 0x46) return 'image/gif'
+    if (
+      buf[0] === 0x52 &&
+      buf[1] === 0x49 &&
+      buf[2] === 0x46 &&
+      buf[3] === 0x46 &&
+      buf[8] === 0x57 &&
+      buf[9] === 0x45 &&
+      buf[10] === 0x42 &&
+      buf[11] === 0x50
+    ) return 'image/webp'
+    // AVIF: ftyp box at offset 4, brand 'avif' at 8
+    if (
+      buf[4] === 0x66 &&
+      buf[5] === 0x74 &&
+      buf[6] === 0x79 &&
+      buf[7] === 0x70 &&
+      buf[8] === 0x61 &&
+      buf[9] === 0x76 &&
+      buf[10] === 0x69 &&
+      buf[11] === 0x66
+    ) return 'image/avif'
+  }
+  // SVG: text-y, contains '<svg' near the top
+  const head = buf.slice(0, 512).toString('utf8')
+  if (/<svg[\s>]/i.test(head)) return 'image/svg+xml'
+  return null
+}
+
+function hashHex16(buf) {
+  return createHash('sha256').update(buf).digest('hex').slice(0, 16)
+}
+
+// Strip editor signatures and identifying blocks from SVG markup. Keeps the
+// drawable content (paths, fills, viewBox) but drops:
+//   - <metadata>…</metadata> blocks (RDF/cc:license/dc:* author info)
+//   - <!-- … --> comments (often embed editor name + path on disk)
+//   - inkscape:* / sodipodi:* attributes and namespace declarations
+//   - <sodipodi:namedview>…</sodipodi:namedview> editor state
+//   - the leading <?xml …?> declaration (carries encoding + sometimes editor id)
+function sanitizeSvg(svg) {
+  let s = String(svg)
+  s = s.replace(/<\?xml[^?]*\?>/g, '')
+  s = s.replace(/<!--[\s\S]*?-->/g, '')
+  s = s.replace(/<metadata\b[\s\S]*?<\/metadata>/gi, '')
+  s = s.replace(/<sodipodi:namedview\b[\s\S]*?<\/sodipodi:namedview>/gi, '')
+  s = s.replace(/<sodipodi:namedview\b[^/>]*\/>/gi, '')
+  s = s.replace(/\sxmlns:(inkscape|sodipodi|cc|dc|rdf|sketch)="[^"]*"/gi, '')
+  s = s.replace(/\s(inkscape|sodipodi|sketch):[\w-]+="[^"]*"/gi, '')
+  return s.trim()
+}
+
+// Walk a directory once and build hash → relative-path index. The skill
+// names files <hash>.<ext> so filename === hash, but we still verify by
+// stripping the extension. Existing assets older than this rule (renamed by
+// hand, etc.) are ignored — they wouldn't dedupe correctly anyway.
+async function indexExisting(dir) {
+  const map = new Map()
+  if (!dir || !existsSync(dir)) return map
+  let entries
+  try {
+    entries = await readdir(dir)
+  } catch {
+    return map
+  }
+  for (const name of entries) {
+    const m = /^([0-9a-f]{16})\.([a-z0-9]+)$/i.exec(name)
+    if (!m) continue
+    map.set(m[1], { path: join(dir, name), ext: m[2].toLowerCase() })
+  }
+  return map
+}
+
+async function fetchBuffer(url) {
+  const ac = new AbortController()
+  const t = setTimeout(() => ac.abort(), TIMEOUT)
+  try {
+    const res = await fetch(url, {
+      signal: ac.signal,
+      redirect: 'follow',
+      headers: {
+        // A neutral UA. Some CDNs 403 on default node fetch UA.
+        'user-agent':
+          'Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Mobile/15E148 Safari/604.1',
+        accept: 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
+      },
+    })
+    if (!res.ok) {
+      return { ok: false, status: res.status, reason: res.statusText || `http-${res.status}` }
+    }
+    const ct = (res.headers.get('content-type') || '').split(';')[0].trim().toLowerCase()
+    const ab = await res.arrayBuffer()
+    return { ok: true, status: res.status, contentType: ct, buffer: Buffer.from(ab) }
+  } catch (err) {
+    return { ok: false, status: 0, reason: err?.name === 'AbortError' ? 'timeout' : (err?.message || String(err)) }
+  } finally {
+    clearTimeout(t)
+  }
+}
+
+async function loadInputUrls() {
+  if (INPUT_URLS) return INPUT_URLS
+  const p = resolve(values['inspection-path'])
+  let raw
+  try {
+    raw = await readFile(p, 'utf8')
+  } catch (err) {
+    fail(`cannot read inspection file: ${err.message}`)
+  }
+  let parsed
+  try {
+    parsed = JSON.parse(raw)
+  } catch (err) {
+    fail(`inspection file is not valid JSON: ${err.message}`)
+  }
+  if (!Array.isArray(parsed.imgUrls)) {
+    fail(`inspection file has no imgUrls[] (got ${typeof parsed.imgUrls})`)
+  }
+  return parsed.imgUrls
+}
+
+async function main() {
+  await mkdir(OUTPUT_DIR, { recursive: true })
+
+  let sharp
+  try {
+    sharp = (await import('sharp')).default
+  } catch (err) {
+    process.stderr.write(
+      `[sb-extract-assets] missing dependency: ${err?.message || err}\n` +
+        `Install with: npm i sharp\n`,
+    )
+    process.exit(1)
+  }
+
+  const inputs = await loadInputUrls()
+  log(`processing ${inputs.length} url(s) into ${OUTPUT_DIR} (target=${TARGET})`)
+
+  // Build hash index from output_dir (idempotent reruns + cross-page dedupe
+  // within the same project) and from --existing-assets-dir if provided.
+  const localIdx = await indexExisting(OUTPUT_DIR)
+  const externalIdx = await indexExisting(EXISTING_DIR)
+  // Merge: external first, local overrides (we prefer assets already saved
+  // in this project's output_dir).
+  const knownHashes = new Map([...externalIdx, ...localIdx])
+
+  // Dedupe input by URL — orchestrator may pass the same URL multiple times
+  // (e.g. same hero referenced from <img> and CSS background-image).
+  const byUrl = new Map()
+  for (const item of inputs) {
+    if (!item || typeof item.url !== 'string') continue
+    if (!byUrl.has(item.url)) byUrl.set(item.url, item)
+  }
+
+  const assets = {}
+  const reuseReport = []
+  const failed = []
+
+  for (const item of byUrl.values()) {
+    const url = item.url
+    const context = item.context || ''
+    const alt = item.alt || ''
+
+    // Skip data: URIs — they're already inline and have no metadata to strip
+    // beyond what the source HTML already exposed. Pass through as failed
+    // with a clear reason so the orchestrator can decide to keep them inline.
+    if (url.startsWith('data:')) {
+      failed.push({ url, status: 0, reason: 'data-uri-skipped' })
+      continue
+    }
+    if (!/^https?:\/\//i.test(url)) {
+      failed.push({ url, status: 0, reason: 'unsupported-scheme' })
+      continue
+    }
+
+    const fetched = await fetchBuffer(url)
+    if (!fetched.ok) {
+      log(`✗ ${url} — ${fetched.status} ${fetched.reason}`)
+      failed.push({ url, status: fetched.status, reason: fetched.reason })
+      continue
+    }
+
+    let mime = MIME_EXT[fetched.contentType] ? fetched.contentType : sniffMime(fetched.buffer)
+    if (!mime || !MIME_EXT[mime]) {
+      // Fallback: trust the URL's extension if it matches something we know
+      const urlExt = (extname(new URL(url).pathname) || '').slice(1).toLowerCase()
+      const guessedMime = Object.entries(MIME_EXT).find(([, e]) => e === urlExt)?.[0]
+      if (guessedMime) mime = guessedMime
+    }
+    if (!mime || !MIME_EXT[mime]) {
+      failed.push({ url, status: fetched.status, reason: `unsupported-mime:${fetched.contentType || 'unknown'}` })
+      continue
+    }
+
+    if (fetched.buffer.length > LARGE_WARN_BYTES) {
+      log(`⚠ ${url} is ${(fetched.buffer.length / 1024 / 1024).toFixed(1)}MB — consider optimizing`)
+    }
+
+    const originalFilename = basename(new URL(url).pathname) || 'asset'
+
+    // ---- SVG path ----
+    if (mime === 'image/svg+xml') {
+      const cleaned = sanitizeSvg(fetched.buffer.toString('utf8'))
+      const cleanedBuf = Buffer.from(cleaned, 'utf8')
+      const hash = hashHex16(cleanedBuf)
+
+      if (TARGET === 'wp') {
+        // Inline mode: emit the sanitized markup directly. No file written.
+        // The orchestrator's WP builder splices `assets[url].inline` into HTML.
+        assets[url] = {
+          inline: cleaned,
+          hash,
+          ext: 'svg',
+          bytes: cleanedBuf.length,
+          originalFilename,
+          strippedMetadata: true,
+          context,
+          alt,
+        }
+        log(`✓ ${url} — svg inline (wp), ${cleanedBuf.length}B`)
+        continue
+      }
+
+      // target=shopify: save as file, dedupe by hash like rasters
+      const known = knownHashes.get(hash)
+      if (known) {
+        assets[url] = {
+          localPath: known.path,
+          hash,
+          ext: known.ext,
+          bytes: cleanedBuf.length,
+          originalFilename,
+          strippedMetadata: true,
+          reusedFrom: known.path,
+          context,
+          alt,
+        }
+        reuseReport.push({ url, reusedFrom: known.path, reason: 'identical-content-hash' })
+        log(`↻ ${url} — reused ${basename(known.path)}`)
+        continue
+      }
+      const localPath = join(OUTPUT_DIR, `${hash}.svg`)
+      await writeFile(localPath, cleanedBuf)
+      knownHashes.set(hash, { path: localPath, ext: 'svg' })
+      assets[url] = {
+        localPath,
+        hash,
+        ext: 'svg',
+        bytes: cleanedBuf.length,
+        originalFilename,
+        strippedMetadata: true,
+        reusedFrom: null,
+        context,
+        alt,
+      }
+      log(`✓ ${url} — svg saved ${hash}.svg`)
+      continue
+    }
+
+    // ---- Raster path (sharp re-encode) ----
+    const ext = MIME_EXT[mime]
+    let stripped
+    let strippedMetadata = true
+    try {
+      // .rotate() bakes EXIF orientation into pixels BEFORE we drop the
+      // EXIF block. Without this an iPhone portrait photo would render
+      // sideways once metadata is gone.
+      // toFormat(<ext>) re-encodes; sharp does NOT preserve metadata by
+      // default — that's the strip. We do NOT call .withMetadata().
+      // .keepIccProfile() (sharp ≥ 0.33) preserves color management for
+      // calibrated screens. Older sharps fall back gracefully because the
+      // method-chain returns the instance.
+      const sharpFormat = ext === 'jpg' ? 'jpeg' : ext
+      let pipeline = sharp(fetched.buffer).rotate()
+      if (typeof pipeline.keepIccProfile === 'function') {
+        pipeline = pipeline.keepIccProfile()
+      }
+      stripped = await pipeline.toFormat(sharpFormat).toBuffer()
+    } catch (err) {
+      // Sharp can't decode (corrupt, unsupported variant, ICO, etc.) — save
+      // raw bytes and flag. Better to ship a possibly-leaky asset than to
+      // black-hole it; the orchestrator can decide whether to drop it.
+      log(`! ${url} — sharp decode failed (${err.message}), saving raw`)
+      stripped = fetched.buffer
+      strippedMetadata = false
+    }
+
+    const hash = hashHex16(stripped)
+    const known = knownHashes.get(hash)
+    if (known) {
+      assets[url] = {
+        localPath: known.path,
+        hash,
+        ext: known.ext,
+        bytes: stripped.length,
+        originalFilename,
+        strippedMetadata,
+        reusedFrom: known.path,
+        context,
+        alt,
+      }
+      reuseReport.push({ url, reusedFrom: known.path, reason: 'identical-content-hash' })
+      log(`↻ ${url} — reused ${basename(known.path)}`)
+      continue
+    }
+
+    const localPath = join(OUTPUT_DIR, `${hash}.${ext}`)
+    await writeFile(localPath, stripped)
+    knownHashes.set(hash, { path: localPath, ext })
+    assets[url] = {
+      localPath,
+      hash,
+      ext,
+      bytes: stripped.length,
+      originalFilename,
+      strippedMetadata,
+      reusedFrom: null,
+      context,
+      alt,
+    }
+    log(`✓ ${url} — saved ${hash}.${ext} (${stripped.length}B${strippedMetadata ? '' : ', RAW'})`)
+  }
+
+  const result = {
+    target: TARGET,
+    outputDir: OUTPUT_DIR,
+    assets,
+    reuseReport,
+    failed,
+  }
+
+  const mapPath = join(OUTPUT_DIR, 'assets-map.json')
+  await writeFile(mapPath, JSON.stringify(result, null, 2))
+  log(`wrote ${mapPath}`)
+
+  process.stdout.write(JSON.stringify(result))
+}
+
+main().catch((err) => {
+  process.stderr.write(`[sb-extract-assets] fatal: ${err?.stack ? err.stack : err}\n`)
+  process.exit(1)
+})

package/templates/skills/sb-extract-assets/scripts/tests/test-extract-assets.mjs

@@ -0,0 +1,112 @@
+#!/usr/bin/env node
+// test-extract-assets.mjs — Smoke tests that don't require sharp or network.
+// Validates: --help works, missing/invalid args fail with exit code 2, syntax loads.
+// Network + sharp integration tests belong in a separate harness.
+
+import { spawnSync } from 'node:child_process'
+import { fileURLToPath } from 'node:url'
+import { dirname, resolve } from 'node:path'
+import { strict as assert } from 'node:assert'
+
+const here = dirname(fileURLToPath(import.meta.url))
+const SCRIPT = resolve(here, '..', 'extract-assets.mjs')
+
+let passed = 0
+let failed = 0
+
+function test(name, fn) {
+  try {
+    fn()
+    process.stdout.write(`ok - ${name}\n`)
+    passed++
+  } catch (err) {
+    process.stdout.write(`not ok - ${name}\n  ${err.message}\n`)
+    failed++
+  }
+}
+
+test('--help exits 0 and prints usage', () => {
+  const r = spawnSync('node', [SCRIPT, '--help'], { encoding: 'utf8' })
+  assert.equal(r.status, 0, `exit code was ${r.status}`)
+  assert.match(r.stdout, /extract-assets\.mjs/)
+  assert.match(r.stdout, /--inspection-path/)
+  assert.match(r.stdout, /--img-urls/)
+  assert.match(r.stdout, /--output-dir/)
+  assert.match(r.stdout, /--target/)
+  assert.match(r.stdout, /--existing-assets-dir/)
+})
+
+test('missing --output-dir exits 2', () => {
+  const r = spawnSync('node', [SCRIPT, '--img-urls', '[]'], { encoding: 'utf8' })
+  assert.equal(r.status, 2, `exit code was ${r.status}`)
+  assert.match(r.stderr, /missing --output-dir/)
+})
+
+test('missing input source exits 2', () => {
+  const r = spawnSync('node', [SCRIPT, '--output-dir', '/tmp/sb-extract-test'], { encoding: 'utf8' })
+  assert.equal(r.status, 2, `exit code was ${r.status}`)
+  assert.match(r.stderr, /missing input/)
+})
+
+test('both --inspection-path and --img-urls exits 2', () => {
+  const r = spawnSync(
+    'node',
+    [SCRIPT, '--output-dir', '/tmp/sb-extract-test', '--img-urls', '[]', '--inspection-path', '/tmp/x.json'],
+    { encoding: 'utf8' },
+  )
+  assert.equal(r.status, 2, `exit code was ${r.status}`)
+  assert.match(r.stderr, /exactly one/)
+})
+
+test('invalid --target exits 2', () => {
+  const r = spawnSync(
+    'node',
+    [SCRIPT, '--output-dir', '/tmp/sb-extract-test', '--img-urls', '[]', '--target', 'wix'],
+    { encoding: 'utf8' },
+  )
+  assert.equal(r.status, 2, `exit code was ${r.status}`)
+  assert.match(r.stderr, /invalid --target/)
+})
+
+test('non-numeric --large-warn-mb exits 2', () => {
+  const r = spawnSync(
+    'node',
+    [SCRIPT, '--output-dir', '/tmp/sb-extract-test', '--img-urls', '[]', '--large-warn-mb', 'foo'],
+    { encoding: 'utf8' },
+  )
+  assert.equal(r.status, 2, `exit code was ${r.status}`)
+  assert.match(r.stderr, /numeric/)
+})
+
+test('malformed --img-urls JSON exits 2', () => {
+  const r = spawnSync(
+    'node',
+    [SCRIPT, '--output-dir', '/tmp/sb-extract-test', '--img-urls', '{not-json'],
+    { encoding: 'utf8' },
+  )
+  assert.equal(r.status, 2, `exit code was ${r.status}`)
+  assert.match(r.stderr, /not valid JSON/)
+})
+
+test('non-array --img-urls exits 2', () => {
+  const r = spawnSync(
+    'node',
+    [SCRIPT, '--output-dir', '/tmp/sb-extract-test', '--img-urls', '{"foo":"bar"}'],
+    { encoding: 'utf8' },
+  )
+  assert.equal(r.status, 2, `exit code was ${r.status}`)
+  assert.match(r.stderr, /must be a JSON array|not valid JSON/)
+})
+
+test('missing inspection file exits 2', () => {
+  const r = spawnSync(
+    'node',
+    [SCRIPT, '--output-dir', '/tmp/sb-extract-test', '--inspection-path', '/tmp/does-not-exist-xyz.json'],
+    { encoding: 'utf8' },
+  )
+  assert.equal(r.status, 2, `exit code was ${r.status}`)
+  assert.match(r.stderr, /not found/)
+})
+
+process.stdout.write(`\n${passed} passed, ${failed} failed\n`)
+process.exit(failed === 0 ? 0 : 1)