sbuilder-mcp 0.17.1 → 0.18.0

package/CHANGELOG.md

@@ -6,6 +6,19 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.18.0] - 2026-09-10
+
+### Added
+- sb_import and sb_import_site can now bring over a `<video>` element, an embedded YouTube, Vimeo, Google Map or SoundCloud player, and an `<hr>` divider, instead of silently dropping every iframe as unsupported.
+
+### Fixed
+- sb_import and sb_import_site no longer capture a source site's own header or footer navigation when it sits inside a wrapper element rather than directly under `<body>`, and no longer mistake a footer marked only by a CSS class (with no `<footer>` tag) for page content.
+- sb_import_site no longer imports the same page twice under two different slugs when the source declares a `<link rel="canonical">` pointing at a URL already in the plan, or when a sitemap or crawl lists per-language copies of the same page (`/about`, `/en/about`, `/vi/about`); the entry page's own language is kept.
+- sb_import_site no longer imports pagination pages such as `/blog/page/2` as separate pages.
+- sb_import_site now honors the site's robots.txt Disallow rules while crawling or reading a sitemap, except for the URL the caller explicitly named.
+- sb_import and sb_import_site no longer duplicate a nested list's items: a `<ul>` inside an `<li>` was previously captured once inside its parent item's text and again as its own list item.
+- sb_import and sb_import_site now skip any element marked `aria-hidden="true"`, so carousel clones and hidden mobile-menu copies no longer come through as duplicated content.
+
 ## [0.17.1] - 2026-09-10
 
 ### Fixed

package/CHANGELOG.vi.md

@@ -6,6 +6,19 @@ Mọi thay đổi đáng chú ý của dự án được ghi lại trong file n
 Định dạng dựa trên [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 và dự án tuân theo [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.18.0] - 2026-09-10
+
+### Added
+- sb_import và sb_import_site giờ có thể mang theo một element `<video>`, một player nhúng YouTube, Vimeo, Google Map hoặc SoundCloud, và một divider `<hr>`, thay vì âm thầm bỏ qua mọi iframe vì không hỗ trợ.
+
+### Fixed
+- sb_import và sb_import_site giờ không còn lấy nhầm header hoặc footer điều hướng của chính site nguồn khi nó nằm trong một element bọc ngoài thay vì là con trực tiếp của `<body>`, và không còn nhầm một footer chỉ được đánh dấu bằng class CSS (không có thẻ `<footer>`) là nội dung trang.
+- sb_import_site giờ không còn import cùng một trang hai lần dưới hai slug khác nhau khi trang nguồn khai báo `<link rel="canonical">` trỏ tới một URL đã có trong kế hoạch, hoặc khi sitemap/crawl liệt kê các bản dịch theo ngôn ngữ của cùng một trang (`/about`, `/en/about`, `/vi/about`); ngôn ngữ của trang nhập vào được giữ lại.
+- sb_import_site giờ không còn import các trang phân trang như `/blog/page/2` thành các trang riêng.
+- sb_import_site giờ tuân theo các quy tắc Disallow trong robots.txt của site khi crawl hoặc đọc sitemap, trừ URL mà người gọi đã chỉ định rõ ràng.
+- sb_import và sb_import_site giờ không còn nhân đôi các mục của một list lồng nhau: một `<ul>` bên trong `<li>` trước đây bị lấy một lần trong text của mục cha và lấy lại lần nữa như một mục list riêng.
+- sb_import và sb_import_site giờ bỏ qua mọi element được đánh dấu `aria-hidden="true"`, để các bản sao carousel và bản sao ẩn của menu mobile không còn bị lấy thành nội dung trùng lặp.
+
 ## [0.17.1] - 2026-09-10
 
 ### Fixed

package/dist/domains/site/discover.js

@@ -39,6 +39,79 @@ const NOT_CONTENT = [
     // as often /blog/tag/x as /tag/x.
     '/tag/', '/tags/', '/author/', '/authors/',
 ];
+/**
+ * PAGE 2 OF A LIST IS NOT A PAGE.
+ *
+ * `/blog/page/2` is the same design as `/blog` holding the next twenty records,
+ * and this platform renders a list from its own catalogue rather than from
+ * somebody else's pagination. Importing them spends the page budget on repeats
+ * of a layout already taken. Narrow on purpose: `/blog/2024` is a year archive
+ * and a real page, so only an explicit `page` segment counts.
+ */
+const PAGINATION = /\/(?:page|pages|p)\/\d+(?:\/|$)/i;
+/** A leading path segment that is a language tag: `/en`, `/vi`, `/en-us`, `/zh-hans`. */
+const LOCALE_SEG = /^[a-z]{2}(?:-[a-z]{2,4})?$/i;
+/** A path split into the language it is written in and the page it names. */
+function localeOf(path) {
+    const segs = path.split('/').filter(Boolean);
+    if (segs.length > 0 && LOCALE_SEG.test(segs[0])) {
+        return { lang: segs[0].toLowerCase(), key: `/${segs.slice(1).join('/')}` };
+    }
+    return { lang: '', key: path };
+}
+/**
+ * ONE PAGE PER PAGE, not one per language.
+ *
+ * A multilingual site lists every translation in its sitemap, so `/about`,
+ * `/en/about` and `/vi/about` all arrive and all describe the same page. Import
+ * them and the merchant gets the same content three times under three slugs,
+ * with the page budget spent on translations of a page already taken — and this
+ * platform has a translations surface for exactly that job.
+ *
+ * FOLDED ONLY ON A CONFLICT. A rule that simply dropped every `/xx/` prefix
+ * would empty the plan for a site that serves ALL its pages under one — which
+ * nodejs.org does, everything under `/en`. So a group forms only when two URLs
+ * name the same page in different languages, which means the list can never
+ * come back shorter than the number of distinct pages.
+ *
+ * The entry's own language wins, because that is the one the caller pointed at.
+ */
+function foldLocales(kept, entry, skip) {
+    const entryLang = localeOf(pathOf(entry)).lang;
+    const groups = new Map();
+    for (const f of kept) {
+        const { key } = localeOf(pathOf(f.url));
+        const g = groups.get(key);
+        if (g)
+            g.push(f);
+        else
+            groups.set(key, [f]);
+    }
+    const out = [];
+    for (const group of groups.values()) {
+        if (group.length === 1) {
+            out.push(group[0]);
+            continue;
+        }
+        const langs = group.map((f) => localeOf(pathOf(f.url)).lang);
+        if (new Set(langs).size === 1) {
+            // Same language, same key: not translations at all, so nothing to choose
+            // between. `/en/a` and `/en/a` cannot both be here — `seen` dropped that —
+            // so this is a key collision the fold has no opinion about.
+            out.push(...group);
+            continue;
+        }
+        const pick = group.find((f) => f.url === entry) ??
+            group.find((f) => localeOf(pathOf(f.url)).lang === entryLang) ??
+            group.find((f) => localeOf(pathOf(f.url)).lang === '') ??
+            group[0];
+        for (const f of group)
+            if (f !== pick)
+                skip('other-locale');
+        out.push(pick);
+    }
+    return out;
+}
 /**
  * Is this path the site's plumbing rather than one of its pages?
  *
@@ -167,6 +240,75 @@ export function robotsSitemaps(txt) {
     }
     return out;
 }
+/**
+ * The `User-agent: *` group's rules.
+ *
+ * HONOURED, not read for interest. A merchant importing their own site is one
+ * caller; the other points this at somebody else's server, and a tool that
+ * fetches a dozen pages should obey the file that exists to say which. It is
+ * also an accuracy win on its own terms — what a site disallows is almost always
+ * its plumbing, and the plumbing list here is a guess where robots.txt is the
+ * site's own answer.
+ *
+ * `Allow` is collected too, because the standard resolves a conflict by the
+ * LONGEST match and a site that disallows `/blog/` and allows `/blog/public/`
+ * means the second.
+ */
+export function robotsRules(txt) {
+    const allow = [];
+    const disallow = [];
+    let applies = false;
+    for (const line of txt.split(/\r?\n/)) {
+        const clean = line.replace(/#.*$/, '').trim();
+        if (!clean)
+            continue;
+        const m = /^([A-Za-z-]+)\s*:\s*(.*)$/.exec(clean);
+        if (!m)
+            continue;
+        const field = m[1].toLowerCase();
+        const value = m[2].trim();
+        if (field === 'user-agent') {
+            applies = value === '*';
+            continue;
+        }
+        if (!applies || !value)
+            continue;
+        // AN EMPTY `Disallow:` MEANS ALLOW EVERYTHING, and treating it as the empty
+        // prefix would block every path on the site — the one parsing mistake in
+        // this file that turns a polite crawler into a crawler that finds nothing.
+        if (field === 'disallow')
+            disallow.push(value);
+        else if (field === 'allow')
+            allow.push(value);
+    }
+    return { allow, disallow };
+}
+/** Does the site ask a general crawler to leave this path alone? */
+export function blockedByRobots(path, rules) {
+    const match = (rule) => {
+        // `*` and `$` are the two wildcards every major crawler honours. Anything
+        // else in a rule is a literal prefix.
+        const pattern = rule
+            .replace(/[.+?^${}()|[\]\\]/g, '\\$&')
+            .replace(/\*/g, '.*')
+            .replace(/\\\$$/, '$');
+        let re;
+        try {
+            re = new RegExp(`^${pattern}`);
+        }
+        catch {
+            return path.startsWith(rule) ? rule.length : -1;
+        }
+        return re.test(path) ? rule.length : -1;
+    };
+    const deny = Math.max(-1, ...rules.disallow.map(match));
+    if (deny < 0)
+        return false;
+    const permit = Math.max(-1, ...rules.allow.map(match));
+    // LONGEST MATCH WINS, and a tie goes to the crawler — that is what the
+    // standard says and what every implementation does.
+    return deny > permit;
+}
 /** A slug this platform will accept, from a path. The root is the home page. */
 export function slugFor(url, taken) {
     const path = pathOf(url);
@@ -271,6 +413,16 @@ export function choosePages(entry, urls, opts = {}) {
                 skip('not-content');
                 continue;
             }
+            if (PAGINATION.test(path)) {
+                skip('pagination');
+                continue;
+            }
+            // THE ENTRY IS EXEMPT. The caller typed that URL, and a merchant whose own
+            // robots.txt disallows their own home page has still asked for it.
+            if (opts.robots && norm !== entry && blockedByRobots(path, opts.robots)) {
+                skip('robots-disallow');
+                continue;
+            }
         }
         if (new URL(norm).search) {
             // A query string is nearly always a filter, a sort or a page number over
@@ -291,10 +443,11 @@ export function choosePages(entry, urls, opts = {}) {
         seen.add(norm);
         kept.push({ url: norm, from: norm === entry ? 'entry' : f.from });
     }
+    const folded = foldLocales(kept, entry, skip);
     // GROUPS ARE COUNTED BEFORE THE CAP, because the whole point of reporting them
     // is to say what the cap is about to hide.
     const groups = {};
-    for (const f of kept) {
+    for (const f of folded) {
         const seg = pathOf(f.url).split('/').filter(Boolean)[0];
         if (seg)
             groups[seg] = (groups[seg] ?? 0) + 1;
@@ -303,7 +456,7 @@ export function choosePages(entry, urls, opts = {}) {
         if (groups[k] < 3)
             delete groups[k];
     const depthOf = (u) => pathOf(u).split('/').filter(Boolean).length;
-    kept.sort((a, b) => {
+    folded.sort((a, b) => {
         if (a.url === entry)
             return -1;
         if (b.url === entry)
@@ -311,11 +464,11 @@ export function choosePages(entry, urls, opts = {}) {
         const d = depthOf(a.url) - depthOf(b.url);
         return d !== 0 ? d : a.url.localeCompare(b.url);
     });
-    const over = Math.max(0, kept.length - maxPages);
+    const over = Math.max(0, folded.length - maxPages);
     if (over > 0)
         skipped['over-page-limit'] = over;
     const taken = new Set();
-    const pages = kept.slice(0, maxPages).map((f) => {
+    const pages = folded.slice(0, maxPages).map((f) => {
         const slug = slugFor(f.url, taken);
         return { ...f, slug, name: nameFor(slug), depth: depthOf(f.url) };
     });
@@ -342,6 +495,8 @@ export function canonFor(entry) {
             return null;
         if (isPlumbing(path))
             return null;
+        if (PAGINATION.test(path))
+            return null;
         return norm;
     };
 }

package/dist/domains/site/importmap.js

@@ -94,6 +94,45 @@ function one(c, t) {
                 },
             };
         }
+        case 'divider': {
+            return { type: 'divider', style: { width: '100%' } };
+        }
+        case 'video': {
+            if (!c.src)
+                return null;
+            // NO STYLE OF OUR OWN. Every media element here already seeds
+            // `width: 100%` + `height: fit-content` and `google-map` seeds a height
+            // per breakpoint; writing a literal over that detaches the node from the
+            // element's own responsive answer to be less correct than it.
+            //
+            // `videoRatio` is seeded 16 / 9, which is right for almost every file a
+            // page embeds — and markup that does not state the real ratio is not
+            // something to guess a crop from.
+            return {
+                type: 'video',
+                specials: { videoSrc: c.src, ...(c.poster ? { poster: c.poster } : {}) },
+            };
+        }
+        case 'embed': {
+            // AN EMBED IS A PROVIDER, NOT A URL. `youtube` and `vimeo` store the ID
+            // alone — handing them a whole watch URL renders nothing — while
+            // `google-map` takes the embed URL its own hint asks for and `soundcloud`
+            // takes a track URL under a different key again.
+            if (c.provider === 'youtube' || c.provider === 'vimeo') {
+                if (!c.videoId)
+                    return null;
+                return { type: c.provider, specials: { videoId: c.videoId } };
+            }
+            if (!c.src)
+                return null;
+            if (c.provider === 'map') {
+                return { type: 'google-map', specials: { src: c.src, mapType: 'location' } };
+            }
+            if (c.provider === 'soundcloud') {
+                return { type: 'soundcloud', specials: { trackUrl: c.src } };
+            }
+            return null;
+        }
         case 'image': {
             if (!c.src)
                 return null;

package/dist/tools/importpage.js

@@ -5,7 +5,7 @@ import { uploadMedia } from '../transport/media.js';
 import { addSubtree } from '../domains/site/builder.js';
 import { middleEnd } from '../domains/site/traps.js';
 import { toSpecs, tokensFromPage, imageSources, rehostImages, } from '../domains/site/importmap.js';
-import { canonFor, choosePages, normalizeUrl, robotsSitemaps, sitemapUrls, } from '../domains/site/discover.js';
+import { canonFor, choosePages, normalizeUrl, robotsRules, robotsSitemaps, sitemapUrls, } from '../domains/site/discover.js';
 import { loadSource } from '../transport/pages.js';
 import { PageDoc } from '../domains/site/document.js';
 import { request } from '../transport/http.js';
@@ -48,17 +48,9 @@ async function fetchForeign(ctx, url) {
  * One fetch and no browser, and it lists pages nothing links to — which is why
  * it is tried before the crawl rather than after it.
  */
-export async function fromSitemap(ctx, entry) {
+export async function fromSitemap(ctx, entry, declaredIn) {
     const origin = new URL(entry).origin;
-    const declared = new Set();
-    // ROBOTS.TXT IS WHERE A SITE SAYS WHERE ITS SITEMAP REALLY IS, and plenty of
-    // real ones are not at /sitemap.xml — a shop platform names
-    // /sitemap_products_1.xml, a CMS a dated path. Guessing only the default is how
-    // a site with a perfectly good sitemap gets crawled instead.
-    const robots = await fetchForeign(ctx, `${origin}/robots.txt`);
-    if (robots)
-        for (const u of robotsSitemaps(robots))
-            declared.add(u);
+    const declared = new Set(declaredIn);
     for (const guess of ['/sitemap.xml', '/sitemap_index.xml', '/sitemap-index.xml']) {
         declared.add(`${origin}${guess}`);
     }
@@ -132,20 +124,37 @@ async function existingPages(ctx, siteId) {
  * `include` or `max_pages`, not a slower way to find the same list.
  */
 async function discoverSite(ctx, entry, opts) {
-    const listed = await fromSitemap(ctx, entry);
+    // ROBOTS.TXT ANSWERS TWO QUESTIONS AND IS FETCHED ONCE. Where the sitemap
+    // really is — plenty are not at /sitemap.xml, a shop platform names
+    // /sitemap_products_1.xml and a CMS a dated path — and which paths a general
+    // crawler is asked to leave alone.
+    const origin = new URL(entry).origin;
+    const txt = await fetchForeign(ctx, `${origin}/robots.txt`);
+    const robots = txt ? robotsRules(txt) : undefined;
+    const listed = await fromSitemap(ctx, entry, txt ? robotsSitemaps(txt) : []);
     if (listed)
-        return { source: 'sitemap', urls: listed, titles: new Map(), visited: 0 };
+        return { source: 'sitemap', urls: listed, titles: new Map(), visited: 0, robots, aliases: 0 };
     const crawled = await crawlLinks(entry, {
         depth: opts.depth ?? 1,
         maxVisits: opts.maxVisits ?? 24,
         canon: canonFor(entry),
     });
-    return {
-        source: 'links',
-        urls: crawled.urls.map((u) => ({ url: u, from: u === entry ? 'entry' : 'links' })),
-        titles: crawled.titles,
-        visited: crawled.visited,
-    };
+    // A PAGE THAT NAMES ANOTHER ADDRESS AS ITS OWN IS THAT PAGE. Folded here, on
+    // the crawl path, where the answer is already in hand — the sitemap path has
+    // no canonical until the page is opened, and the import pass folds that one.
+    const seen = new Set();
+    const urls = [];
+    let aliases = 0;
+    for (const u of crawled.urls) {
+        const real = crawled.canonical.get(u) ?? u;
+        if (real !== u)
+            aliases += 1;
+        if (seen.has(real))
+            continue;
+        seen.add(real);
+        urls.push({ url: real, from: real === entry ? 'entry' : 'links' });
+    }
+    return { source: 'links', urls, titles: crawled.titles, visited: crawled.visited, robots, aliases };
 }
 export function registerImportTools(server, ctx, session) {
     server.registerTool('sb_import', {
@@ -321,6 +330,7 @@ export function registerImportTools(server, ctx, session) {
             maxPages: max_pages,
             include,
             exclude,
+            robots: found.robots,
         });
         // A CRAWL ALREADY READ THE TITLE. `nameFor` derives a name from the slug
         // because a sitemap offers nothing else, but the link crawl opened every
@@ -394,7 +404,9 @@ export function registerImportTools(server, ctx, session) {
                 discovered_by: found.source,
                 ...(found.visited ? { pages_read_to_find_them: found.visited } : {}),
                 pages: plan.pages.map((p) => ({ url: p.url, slug: p.slug, name: p.name, ...lands(p) })),
-                ...(Object.keys(plan.skipped).length ? { skipped: plan.skipped } : {}),
+                ...(Object.keys(plan.skipped).length || found.aliases
+                    ? { skipped: { ...plan.skipped, ...(found.aliases ? { 'canonical-alias': found.aliases } : {}) } }
+                    : {}),
                 ...(unlistable
                     ? {
                         landing_unknown: `This site's own pages could not be read (${unlistable}), so which of the above ` +
@@ -467,6 +479,13 @@ export function registerImportTools(server, ctx, session) {
         // three pages built, nine not, and no report saying which.
         const built = [];
         const failed = [];
+        // WHAT EACH PAGE SAYS ITS OWN ADDRESS IS. A sitemap cannot tell you that
+        // two of its entries are one page — only the page can, and only once it is
+        // open. Measured: modelcontextprotocol.io's home page declares a dated
+        // docs path as its canonical, so `/` and that path are the same content
+        // under two slugs, and nothing in the plan looks wrong.
+        const identities = new Set();
+        const aliased = [];
         let lastOpened = '';
         for (const p of plan.pages) {
             const shot = byUrl.get(p.url);
@@ -474,6 +493,12 @@ export function registerImportTools(server, ctx, session) {
                 failed.push({ url: p.url, why: shot ? shot.why : 'was not read' });
                 continue;
             }
+            const identity = normalizeUrl(shot.result.canonical ?? p.url) ?? p.url;
+            if (identities.has(identity)) {
+                aliased.push({ url: p.url, same_as: identity });
+                continue;
+            }
+            identities.add(identity);
             try {
                 const sections = rehosted.size > 0 ? rehostImages(shot.result.sections, rehosted) : shot.result.sections;
                 const specs = toSpecs(sections, tokens);
@@ -557,7 +582,10 @@ export function registerImportTools(server, ctx, session) {
             discovered_by: found.source,
             built,
             ...(failed.length ? { failed } : {}),
-            ...(Object.keys(plan.skipped).length ? { skipped: plan.skipped } : {}),
+            ...(aliased.length ? { same_page: aliased } : {}),
+            ...(Object.keys(plan.skipped).length || found.aliases
+                ? { skipped: { ...plan.skipped, ...(found.aliases ? { 'canonical-alias': found.aliases } : {}) } }
+                : {}),
             images: {
                 copied: rehosted.size,
                 ...(failedImages.length ? { failed: failedImages } : {}),

package/dist/vision/capture.js

@@ -63,10 +63,31 @@ function capturePage(limits) {
         const bordered = cs.borderStyle !== '' && cs.borderStyle !== 'none' && parseFloat(cs.borderWidth || '0') > 0;
         return cs.display !== 'inline' && (filled || bordered);
     };
+    // IFRAME IS NOT HERE ANY MORE. It was, and it took every embedded video, every
+    // map and every audio player with it — silently, as a skip count. The platform
+    // has `video`, `youtube`, `vimeo`, `soundcloud` and `google-map`; a hero video
+    // and a contact page's map are ordinary things to import, and they were the
+    // one kind of content that could not survive the trip at all.
     const IGNORE = new Set([
-        'SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'SVG', 'PATH', 'IFRAME', 'CANVAS',
+        'SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'SVG', 'PATH', 'CANVAS',
         'NAV', 'FORM', 'INPUT', 'SELECT', 'TEXTAREA', 'BUTTON',
     ]);
+    /** The provider and id behind an embed URL, or null if this platform has no element for it. */
+    const embedOf = (raw) => {
+        const u = raw.split('?')[0];
+        let m = /(?:youtube(?:-nocookie)?\.com\/(?:embed|v|shorts)\/|youtu\.be\/)([A-Za-z0-9_-]{6,})/.exec(u);
+        if (m)
+            return { provider: 'youtube', videoId: m[1] };
+        m = /player\.vimeo\.com\/video\/(\d+)/.exec(u);
+        if (m)
+            return { provider: 'vimeo', videoId: m[1] };
+        if (/(?:google\.[a-z.]+|maps\.google\.[a-z.]+)\/maps?\/embed/.test(u)) {
+            return { provider: 'map', src: raw };
+        }
+        if (/w\.soundcloud\.com\/player/.test(u))
+            return { provider: 'soundcloud', src: raw };
+        return null;
+    };
     /**
      * The renderable content under one section, AS A TREE.
      *
@@ -83,6 +104,77 @@ function capturePage(limits) {
      * merely wraps is not a design decision and reproducing it would nest the
      * result ten deep for nothing.
      */
+    // THE SOURCE'S OWN HEADER AND FOOTER ARE NEVER WANTED. The target site has its
+    // own, as shared globals, and importing somebody else's navigation onto a
+    // storefront is a second menu pointing at a different website.
+    //
+    // NOT NAMED `chrome`: that is a BROWSER GLOBAL (the extension API), so a
+    // `const chrome` declared in a nested scope left every OTHER scope resolving
+    // the name to `window.chrome` — `chrome.has is not a function`, thrown inside
+    // `evaluate`, which kills the whole capture. The same shape as the closure
+    // trap this file already carries, reached from the opposite direction.
+    //
+    // PAGE-LEVEL, BY THE SPEC'S OWN DEFINITION rather than by depth. This used to
+    // ask whether the element was a DIRECT child of `<body>`, which almost no real
+    // site satisfies — one wrapper div is enough — so blender.org's footer came
+    // through as eleven sections of link columns and the page a merchant asked for
+    // was its site map. `<header>` and `<footer>` belong to their nearest
+    // SECTIONING ancestor (article, aside, nav, section), so one with none of
+    // those above it is the page's, however deeply it is wrapped; one inside an
+    // `<article>` is that article's byline, and one inside a `<section>` is the
+    // hero the old comment was right to protect. `<main>` is not sectioning
+    // content, so it does not shield a footer.
+    const sectioning = Array.from(document.querySelectorAll('article, section, aside, nav'));
+    // A CLASS NAME IS EVIDENCE FOR A FOOTER AND NOT FOR A HEADER, and the
+    // asymmetry is the whole point. blender.org marks its site map
+    // `<div class="footer-navigation">` — no `<footer>` tag anywhere near it — so
+    // the spec rule alone let eleven sections of somebody else's links through as
+    // the page a merchant asked for. The same trick on the header side would eat
+    // HEROES: blender's own first band is `<div class="hero header-size-large">`,
+    // and losing the first thing on a landing page costs more than a stray footer.
+    // A header is caught by its tag or its ARIA role, and its links are `<nav>`,
+    // which is ignored already.
+    //
+    // The token must START with `footer` (footer, footer-note, footer__inner) so
+    // `card-footer` inside an ordinary div is not swept up with it.
+    const footerish = (el) => {
+        const words = `${el.getAttribute('id') ?? ''} ${String(el.className ?? '')}`.toLowerCase();
+        for (const w of words.split(/[\s]+/)) {
+            if (!w)
+                continue;
+            if (w === 'colophon' || w === 'site-footer' || w === 'page-footer')
+                return true;
+            if (w === 'footer' || w.indexOf('footer-') === 0 || w.indexOf('footer_') === 0)
+                return true;
+        }
+        return false;
+    };
+    const pageChromeRoots = [];
+    const chromeCandidates = Array.from(document.querySelectorAll('header, footer, [role="banner"], [role="contentinfo"], [class*="footer"], [id*="footer"]'));
+    for (const el of chromeCandidates) {
+        const role = el.getAttribute('role');
+        const isChrome = el.tagName === 'HEADER' ||
+            el.tagName === 'FOOTER' ||
+            role === 'banner' ||
+            role === 'contentinfo' ||
+            footerish(el);
+        if (!isChrome)
+            continue;
+        if (sectioning.some((sec) => sec !== el && sec.contains(el)))
+            continue;
+        // A NESTED ONE ADDS NOTHING: the outer root already covers it, and keeping
+        // both makes the containment test scan the same subtree twice.
+        if (pageChromeRoots.some((c) => c.contains(el)))
+            continue;
+        pageChromeRoots.push(el);
+    }
+    // INSIDE the chrome, not equal to it. A real footer holds `<section>`s, and
+    // the candidate walk deliberately takes the INNERMOST sections — so on
+    // blender.org the candidates were the footer's own link columns, none of which
+    // IS the footer, and eleven sections of somebody else's site map came through
+    // as the page. Asking about containment is the same question the candidate
+    // list already answers for nesting.
+    const inPageChrome = (el) => pageChromeRoots.some((c) => c === el || c.contains(el));
     const leaves = (root) => {
         const walkChildren = (el) => {
             const kids = [];
@@ -104,7 +196,24 @@ function capturePage(limits) {
                 skip('over-node-limit');
                 return [];
             }
+            if (inPageChrome(el)) {
+                skip('page-chrome');
+                return [];
+            }
             const tag = el.tagName;
+            // WHAT THE PAGE ITSELF SAYS IS NOT CONTENT.
+            //
+            // `aria-hidden="true"` is the author's own mark for decoration and for
+            // duplicates — a carousel's cloned slides, the mobile copy of a menu that
+            // the desktop layout also carries, an icon that repeats the label beside
+            // it. Measured on real pages before this: 53 such elements on one, 15 on
+            // another, every one of them walked and some of them captured twice.
+            // Nothing here reads the accessibility tree, so this attribute is the only
+            // place that answer exists.
+            if (el.getAttribute('aria-hidden') === 'true') {
+                skip('aria-hidden');
+                return [];
+            }
             if (IGNORE.has(tag)) {
                 skip(tag.toLowerCase());
                 return [];
@@ -113,6 +222,40 @@ function capturePage(limits) {
                 skip('hidden');
                 return [];
             }
+            // A RULE BETWEEN SECTIONS IS A DESIGN DECISION, and it is one node.
+            if (tag === 'HR') {
+                taken.nodes++;
+                return [{ kind: 'divider' }];
+            }
+            if (tag === 'VIDEO') {
+                const direct = el.getAttribute('src');
+                const source = Array.from(el.querySelectorAll('source'))[0];
+                const src = direct || (source ? source.getAttribute('src') : null);
+                if (!src) {
+                    skip('video-without-src');
+                    return [];
+                }
+                taken.nodes++;
+                const poster = el.getAttribute('poster');
+                return [{
+                        kind: 'video',
+                        src: abs(src),
+                        ...(poster ? { poster: abs(poster) } : {}),
+                    }];
+            }
+            if (tag === 'IFRAME') {
+                const src = el.getAttribute('src');
+                const embed = src ? embedOf(abs(src)) : null;
+                if (!embed) {
+                    // An advert, a tracking pixel, a chat widget, a comment system: real
+                    // pages carry several, and this platform has an element for none of
+                    // them. Counted rather than guessed at.
+                    skip('iframe');
+                    return [];
+                }
+                taken.nodes++;
+                return [{ kind: 'embed', ...embed }];
+            }
             if (HEADINGS.has(tag)) {
                 const text = clean(el.textContent);
                 if (!text)
@@ -175,9 +318,39 @@ function capturePage(limits) {
                 return [];
             }
             if (tag === 'UL' || tag === 'OL') {
-                const items = Array.from(el.querySelectorAll('li'))
-                    .map((li) => clean(li.textContent))
-                    .filter(Boolean);
+                // A NESTED LIST WAS TAKEN TWICE, and the duplication reads as a page
+                // that stutters. `querySelectorAll('li')` returns the nested items as
+                // well as the outer ones, and an outer item's `textContent` ALREADY
+                // contains its sublist — so every nested entry arrived once inside its
+                // parent's line and once again on its own. Measured on a real import:
+                // one section of blender.org repeated five sublists that way.
+                //
+                // Walked by DIRECT children instead, with each item's own words
+                // separated from its sublist's and the sublist flattened after it. This
+                // platform's `list` is flat, so flattening is the honest translation —
+                // and the order a reader sees is preserved.
+                const items = [];
+                const collect = (list) => {
+                    for (const li of Array.from(list.children)) {
+                        if (li.tagName !== 'LI')
+                            continue;
+                        const sublists = Array.from(li.children).filter((c) => c.tagName === 'UL' || c.tagName === 'OL');
+                        let text = clean(li.textContent);
+                        for (const sub of sublists) {
+                            const inner = clean(sub.textContent);
+                            // textContent runs in document order, so a sublist's words are the
+                            // tail of its parent's. Only strip what is actually there.
+                            if (inner && text.length > inner.length && text.slice(-inner.length) === inner) {
+                                text = clean(text.slice(0, text.length - inner.length));
+                            }
+                        }
+                        if (text)
+                            items.push(text);
+                        for (const sub of sublists)
+                            collect(sub);
+                    }
+                };
+                collect(el);
                 if (!items.length)
                     return [];
                 taken.nodes++;
@@ -286,20 +459,10 @@ function capturePage(limits) {
     // document is one candidate too, and keeping THAT would reduce every page to a
     // single band. The finest ones are the page's actual bands.
     candidates = candidates.filter((el) => !candidates.some((o) => o !== el && el.contains(o)));
-    // THE SOURCE'S OWN HEADER AND FOOTER ARE NEVER WANTED. The target site has its
-    // own, as shared globals, and importing somebody else's navigation onto a
-    // storefront is a second menu pointing at a different website. Only the
-    // PAGE-LEVEL ones are dropped — a `<header>` inside a section is a hero, and
-    // excluding those would lose the first thing on most landing pages.
-    const chrome = new Set();
-    for (const el of Array.from(document.body.children)) {
-        if (el.tagName === 'HEADER' || el.tagName === 'FOOTER')
-            chrome.add(el);
-    }
     const build = (from) => {
         const acc = [];
         for (const el of from) {
-            if (chrome.has(el)) {
+            if (inPageChrome(el)) {
                 skip('page-chrome');
                 continue;
             }
@@ -340,7 +503,15 @@ function capturePage(limits) {
         const main = document.querySelectorAll('main')[0] ?? document.body;
         sections = build(Array.from(main.children));
     }
-    return { url: here, title: clean(document.title), sections, skipped };
+    const link = Array.from(document.querySelectorAll('link[rel="canonical"]'))[0];
+    const canonical = link ? (link.getAttribute('href') ?? '') : '';
+    return {
+        url: here,
+        title: clean(document.title),
+        ...(canonical ? { canonical: abs(canonical) } : {}),
+        sections,
+        skipped,
+    };
 }
 /**
  * ONE BROWSER FOR THE WHOLE CALL.
@@ -462,7 +633,20 @@ function linksOnPage() {
             // One unresolvable href must not kill the crawl.
         }
     }
-    return { title: document.title, links };
+    const link = Array.from(document.querySelectorAll('link[rel="canonical"]'))[0];
+    let canonical = '';
+    if (link) {
+        const href = link.getAttribute('href');
+        if (href) {
+            try {
+                canonical = new URL(href, here).href;
+            }
+            catch {
+                canonical = '';
+            }
+        }
+    }
+    return { title: document.title, canonical, links };
 }
 /**
  * Walk a site's own links from one entry page, breadth first.
@@ -479,6 +663,7 @@ export async function crawlLinks(entry, opts) {
     const maxVisits = opts.maxVisits ?? 24;
     const found = new Set([entry]);
     const titles = new Map();
+    const canonical = new Map();
     let visited = 0;
     await withBrowser(async (browser) => {
         let frontier = [entry];
@@ -496,6 +681,14 @@ export async function crawlLinks(entry, opts) {
                     const got = await readPage(browser, url, 1440, (page) => page.evaluate(linksOnPage));
                     if (got.title)
                         titles.set(url, got.title);
+                    if (got.canonical) {
+                        const c = opts.canon(got.canonical);
+                        // A PAGE THAT NAMES ANOTHER ADDRESS AS ITS OWN is that page. Recorded
+                        // rather than acted on here: the crawl still walks this copy for its
+                        // links, and the caller folds the two when it builds the plan.
+                        if (c && c !== url)
+                            canonical.set(url, c);
+                    }
                     // A LEVEL BELOW THE LAST IS WALKED FOR ITS LINKS AND NOT QUEUED: at
                     // `depth` the crawl still wants what that page points at, it just
                     // must not navigate any further.
@@ -514,5 +707,5 @@ export async function crawlLinks(entry, opts) {
             frontier = next;
         }
     });
-    return { urls: [...found], titles, visited };
+    return { urls: [...found], titles, canonical, visited };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sbuilder-mcp",
-  "version": "0.17.1",
+  "version": "0.18.0",
   "description": "MCP server that designs and operates a Store Builder site — pages, data, theme and publish — through the platform's own API and live-edit protocol.",
   "mcpName": "io.github.vuluu2k/sbuilder-mcp",
   "type": "module",