npm - similarbuild - Versions diffs - 0.1.0 - Mend

similarbuild 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (76) hide show

package/templates/skills/sb-crawl-and-list/scripts/tests/test-sitemap-parser.mjs ADDED Viewed

@@ -0,0 +1,157 @@
+#!/usr/bin/env node
+// Tests for lib/sitemap-parser.mjs — pure unit tests, no network.
+import { strict as assert } from 'node:assert'
+import { parseSitemap, fetchAndParseSitemap } from '../lib/sitemap-parser.mjs'
+let passed = 0
+let failed = 0
+async function test(name, fn) {
+  try {
+    await fn()
+    process.stdout.write(`ok - ${name}\n`)
+    passed++
+  } catch (err) {
+    process.stdout.write(`not ok - ${name}\n  ${err.message}\n`)
+    failed++
+  }
+}
+const URLSET_BASIC = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url><loc>https://example.com/</loc></url>
+  <url><loc>https://example.com/products/foo</loc></url>
+  <url><loc>https://example.com/about</loc></url>
+</urlset>`
+const URLSET_CDATA = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset>
+  <url><loc><![CDATA[https://example.com/path?a=1&b=2]]></loc></url>
+</urlset>`
+const URLSET_ENTITIES = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset>
+  <url><loc>https://example.com/path?a=1&amp;b=2&amp;c=3</loc></url>
+</urlset>`
+const SITEMAPINDEX = `<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex>
+  <sitemap><loc>https://example.com/sitemap-1.xml</loc></sitemap>
+  <sitemap><loc>https://example.com/sitemap-2.xml</loc></sitemap>
+</sitemapindex>`
+const URLSET_BOM = '<urlset><url><loc>https://example.com/</loc></url></urlset>'
+const URLSET_FLAT = `<url><loc>https://example.com/a</loc></url><url><loc>https://example.com/b</loc></url>`
+await test('parseSitemap extracts URLs from a basic urlset', () => {
+  const r = parseSitemap(URLSET_BASIC)
+  assert.equal(r.kind, 'urlset')
+  assert.equal(r.malformed, false)
+  assert.deepEqual(r.urlSetUrls, [
+    'https://example.com/',
+    'https://example.com/products/foo',
+    'https://example.com/about',
+  ])
+})
+await test('parseSitemap unwraps CDATA payloads', () => {
+  const r = parseSitemap(URLSET_CDATA)
+  assert.equal(r.kind, 'urlset')
+  assert.deepEqual(r.urlSetUrls, ['https://example.com/path?a=1&b=2'])
+})
+await test('parseSitemap decodes XML entities in <loc>', () => {
+  const r = parseSitemap(URLSET_ENTITIES)
+  assert.deepEqual(r.urlSetUrls, ['https://example.com/path?a=1&b=2&c=3'])
+})
+await test('parseSitemap detects sitemapindex and lists child sitemaps', () => {
+  const r = parseSitemap(SITEMAPINDEX)
+  assert.equal(r.kind, 'index')
+  assert.deepEqual(r.sitemapIndexUrls, [
+    'https://example.com/sitemap-1.xml',
+    'https://example.com/sitemap-2.xml',
+  ])
+  assert.deepEqual(r.urlSetUrls, [])
+})
+await test('parseSitemap strips BOM', () => {
+  const r = parseSitemap(URLSET_BOM)
+  assert.equal(r.kind, 'urlset')
+  assert.deepEqual(r.urlSetUrls, ['https://example.com/'])
+})
+await test('parseSitemap accepts wrapper-less <url><loc> sequences', () => {
+  const r = parseSitemap(URLSET_FLAT)
+  assert.equal(r.kind, 'urlset')
+  assert.equal(r.urlSetUrls.length, 2)
+})
+await test('parseSitemap returns malformed for empty input', () => {
+  const r = parseSitemap('')
+  assert.equal(r.malformed, true)
+  assert.deepEqual(r.urlSetUrls, [])
+})
+await test('parseSitemap returns malformed for non-sitemap XML', () => {
+  const r = parseSitemap('<html><body>not a sitemap</body></html>')
+  assert.equal(r.kind, 'unknown')
+  assert.equal(r.malformed, true)
+})
+await test('fetchAndParseSitemap returns urlset directly', async () => {
+  const fetcher = async () => ({ ok: true, status: 200, text: URLSET_BASIC })
+  const r = await fetchAndParseSitemap('https://example.com/sitemap.xml', fetcher)
+  assert.equal(r.urls.length, 3)
+  assert.deepEqual(r.warnings, [])
+})
+await test('fetchAndParseSitemap recurses into sitemapindex one level', async () => {
+  const fetcher = async (url) => {
+    if (url.endsWith('/sitemap.xml')) return { ok: true, status: 200, text: SITEMAPINDEX }
+    if (url.endsWith('/sitemap-1.xml')) return { ok: true, status: 200, text: URLSET_BASIC }
+    if (url.endsWith('/sitemap-2.xml'))
+      return {
+        ok: true,
+        status: 200,
+        text: '<urlset><url><loc>https://example.com/extra</loc></url></urlset>',
+      }
+    return { ok: false, status: 404 }
+  }
+  const r = await fetchAndParseSitemap('https://example.com/sitemap.xml', fetcher)
+  assert.equal(r.urls.length, 4)
+  assert.deepEqual(r.warnings, [])
+  assert.ok(r.urls.includes('https://example.com/extra'))
+})
+await test('fetchAndParseSitemap warns on child fetch failure but continues', async () => {
+  const fetcher = async (url) => {
+    if (url.endsWith('/sitemap.xml')) return { ok: true, status: 200, text: SITEMAPINDEX }
+    if (url.endsWith('/sitemap-1.xml')) return { ok: true, status: 200, text: URLSET_BASIC }
+    return { ok: false, status: 404 }
+  }
+  const r = await fetchAndParseSitemap('https://example.com/sitemap.xml', fetcher)
+  assert.equal(r.urls.length, 3)
+  assert.equal(r.warnings.length, 1)
+  assert.match(r.warnings[0], /sitemap-child-failed:.*sitemap-2\.xml/)
+})
+await test('fetchAndParseSitemap surfaces top-level fetch failure', async () => {
+  const fetcher = async () => ({ ok: false, status: 503 })
+  const r = await fetchAndParseSitemap('https://example.com/sitemap.xml', fetcher)
+  assert.deepEqual(r.urls, [])
+  assert.equal(r.warnings.length, 1)
+  assert.match(r.warnings[0], /sitemap-fetch-failed:503/)
+})
+await test('fetchAndParseSitemap surfaces malformed top-level XML', async () => {
+  const fetcher = async () => ({ ok: true, status: 200, text: '<html>nope</html>' })
+  const r = await fetchAndParseSitemap('https://example.com/sitemap.xml', fetcher)
+  assert.deepEqual(r.urls, [])
+  assert.deepEqual(r.warnings, ['sitemap-malformed'])
+})
+process.stdout.write(`\n${passed} passed, ${failed} failed\n`)
+process.exit(failed === 0 ? 0 : 1)

package/templates/skills/sb-extract-assets/SKILL.md ADDED Viewed

@@ -0,0 +1,112 @@
+---
+name: sb-extract-assets
+description: Downloads images from inspected URLs, strips identifying metadata (EXIF/XMP/IPTC), renames via content-hash, and dedupes against assets already on disk. Use when the SimilarBuild orchestrator (`/build-page`, `/build-site`, `/clip-section`) requests asset extraction after `sb-inspect-live`, or when the user asks to 'extract assets'.
+---
+# sb-extract-assets
+## Overview
+Takes the `imgUrls` list from `sb-inspect-live` and produces a clean, tracking-resistant asset library: each file is downloaded, **all camera/editor metadata stripped**, **renamed to a content-hash** (`sha256(buffer)[:16].<ext>`), and saved into the project's asset folder. Same binary → same hash → automatic dedupe across pages of a site batch and across reruns.
+The strip is non-negotiable. Cameras, photo editors, CMS plugins, and stock-image platforms inject EXIF/XMP/IPTC blocks that often contain the original photographer's name, GPS coords, software signatures, copyright, and source URLs — all of which leak provenance and make a rebuild trivially traceable to its source. Sharp's default re-encode drops every metadata block while preserving the ICC color profile (so colors render faithfully on calibrated screens). The script never calls `.withMetadata()`.
+Act as a sanitization gate: anything passed downstream to `sb-build-wp` / `sb-build-shopify` is opaque, deduplicated, and safe to ship. The original filename and source URL live only in the local `assets-map.json` for debug, never in the rendered HTML.
+## Inputs
+| Argument             | Required | Default     | Notes                                                                                  |
+| -------------------- | -------- | ----------- | -------------------------------------------------------------------------------------- |
+| `inspection-path`    | yes\*    | —           | Path to `inspection.json` from `sb-inspect-live` (script reads `imgUrls`).             |
+| `img-urls`           | yes\*    | —           | Inline JSON array `[{url, context, alt}]` — alternative to `--inspection-path`.        |
+| `output-dir`         | yes      | —           | Directory to save sanitized assets and `assets-map.json`.                              |
+| `target`             | no       | `wp`        | `wp` (SVGs returned inline-ready) or `shopify` (SVGs saved as files).                  |
+| `existing-assets-dir`| no       | (none)      | Extra directory of already-extracted assets to dedupe against (e.g. sibling pages).    |
+| `large-warn-mb`      | no       | `10`        | Warn (stderr) if a single asset is larger than this.                                   |
+| `timeout`            | no       | `30000`     | Per-request fetch timeout (ms).                                                        |
+\*Exactly one of `--inspection-path` or `--img-urls` must be provided.
+## Output
+A single JSON object printed to stdout AND saved to `{output_dir}/assets-map.json`:
+```json
+{
+  "target": "wp",
+  "outputDir": "/abs/path",
+  "assets": {
+    "https://example.com/cdn/hero@2x.jpg": {
+      "localPath": "/abs/path/a3f9b2c14e8d7f01.jpg",
+      "hash": "a3f9b2c14e8d7f01",
+      "ext": "jpg",
+      "bytes": 184320,
+      "originalFilename": "hero@2x.jpg",
+      "strippedMetadata": true,
+      "reusedFrom": null,
+      "context": "hero",
+      "alt": "Product hero shot"
+    },
+    "https://example.com/icons/cart.svg": {
+      "inline": "<svg ...></svg>",
+      "originalFilename": "cart.svg",
+      "strippedMetadata": true,
+      "context": "header",
+      "alt": ""
+    }
+  },
+  "reuseReport": [
+    { "url": "https://example.com/cdn/hero-mobile.jpg", "reusedFrom": "https://example.com/cdn/hero@2x.jpg", "reason": "identical-content-hash" }
+  ],
+  "failed": [
+    { "url": "https://example.com/missing.png", "status": 404, "reason": "not-found" }
+  ]
+}
+```
+`originalFilename` is recorded **only** in this map for debug/audit. The orchestrator must NOT propagate it into HTML output (alt text, filenames, comments) — doing so re-attaches the source identity the strip just removed. SVG inlining for `target=wp` returns sanitized markup directly under `assets[url].inline`; for `target=shopify` SVGs are saved as files like raster images.
+## Dependencies
+The host project must have `sharp` installed (raster strip + content-hash). Node ≥ 20 (uses native `fetch` + `crypto.subtle`). The SimilarBuild installer handles this.
+## On Activation
+1. **Resolve inputs.** Collect `inspection-path` *or* `img-urls`, plus `output-dir`, `target`, `existing-assets-dir`. Apply defaults above.
+2. **Ensure `output_dir` exists.** `mkdir -p` it. If `assets-map.json` already exists in `output_dir`, the script reads it first so reruns are idempotent (same input → same map).
+3. **Run the script.** From the project root:
+   ```bash
+   node {skill-root}/scripts/extract-assets.mjs \
+     --output-dir "{output_dir}" \
+     --target "{target}" \
+     [--inspection-path "{path}" | --img-urls '<json>'] \
+     [--existing-assets-dir "{path}"]
+   ```
+   The script handles: HEAD probe + size check, fetch into buffer, sharp re-encode (`.rotate()` first to bake in orientation, then re-encode without metadata, keeping ICC), SVG regex sanitization (`<metadata>`, `<!--…-->`, `inkscape:` / `sodipodi:` namespaces), content-hash naming, dedupe against `output_dir` and `existing-assets-dir`, and the inline-vs-file SVG decision based on `target`. See `scripts/extract-assets.mjs --help` for the full flag list.
+4. **Validate the result.** Parse stdout as JSON. If the script exits non-zero, surface stderr to the orchestrator and stop — don't fabricate a partial map.
+5. **Forward `failed[]` unchanged.** A 404, network error, or content-type mismatch is **not** fatal for this skill. The orchestrator decides whether to re-prompt the user, fall back to a placeholder, or skip the asset entirely. The script never invents a substitute.
+6. **Return the JSON unchanged** to the caller. Do not summarize or strip fields — `sb-build-wp` / `sb-build-shopify` consume `assets[].localPath`, `assets[].inline`, and the `failed[]` list directly.
+## Failure modes
+| Symptom                                          | Likely cause                            | What to surface                                                  |
+| ------------------------------------------------ | --------------------------------------- | ---------------------------------------------------------------- |
+| Script exits non-zero                            | Missing `sharp`, malformed inspection JSON, write failure | Pass stderr verbatim. Suggest `npm i sharp` if the message mentions sharp. |
+| URL appears in `failed[]` with `status: 404`     | Source removed/renamed since inspection | Forward unchanged — orchestrator handles fallback.               |
+| URL appears in `failed[]` with `status: 0`       | DNS / connection / timeout              | Forward unchanged. Orchestrator may retry once.                  |
+| `strippedMetadata: false` on an asset            | Sharp couldn't decode (corrupt or unsupported format) | Asset is saved as-is (raw bytes). Orchestrator should treat it as risky and may opt to drop. |
+| Same URL maps to different hash across runs      | Source returned different bytes (CDN A/B, resized) | Treat as authoritative — it's a new asset version.               |
+## Conventions
+- Bare paths (e.g. `scripts/extract-assets.mjs`) resolve from the skill root.
+- `{skill-root}` resolves to this skill's installed directory.
+- `{project-root}` resolves to the project working directory.
+- Filenames are always `<16-hex-hash>.<ext>` — never the original. Extension is derived from the **decoded MIME type**, not the source URL (a `.jpg?v=2` querystring on a PNG body is corrected).