sbuilder-mcp 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/CHANGELOG.md +20 -0
  2. package/CHANGELOG.vi.md +20 -0
  3. package/dist/core/patch.js +32 -2
  4. package/dist/domains/site/discover.js +159 -4
  5. package/dist/domains/site/importmap.js +39 -0
  6. package/dist/tools/importpage.js +49 -21
  7. package/dist/vision/capture.js +257 -23
  8. package/package.json +1 -1
package/CHANGELOG.md CHANGED
@@ -6,6 +6,26 @@ All notable changes to this project are documented in this file.
6
6
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
7
7
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
8
8
 
9
+ ## [0.18.0] - 2026-09-10
10
+
11
+ ### Added
12
+ - sb_import and sb_import_site can now bring over a `<video>` element, an embedded YouTube, Vimeo, Google Map or SoundCloud player, and an `<hr>` divider, instead of silently dropping every iframe as unsupported.
13
+
14
+ ### Fixed
15
+ - sb_import and sb_import_site no longer capture a source site's own header or footer navigation when it sits inside a wrapper element rather than directly under `<body>`, and no longer mistake a footer marked only by a CSS class (with no `<footer>` tag) for page content.
16
+ - sb_import_site no longer imports the same page twice under two different slugs when the source declares a `<link rel="canonical">` pointing at a URL already in the plan, or when a sitemap or crawl lists per-language copies of the same page (`/about`, `/en/about`, `/vi/about`); the entry page's own language is kept.
17
+ - sb_import_site no longer imports pagination pages such as `/blog/page/2` as separate pages.
18
+ - sb_import_site now honors the site's robots.txt Disallow rules while crawling or reading a sitemap, except for the URL the caller explicitly named.
19
+ - sb_import and sb_import_site no longer duplicate a nested list's items: a `<ul>` inside an `<li>` was previously captured once inside its parent item's text and again as its own list item.
20
+ - sb_import and sb_import_site now skip any element marked `aria-hidden="true"`, so carousel clones and hidden mobile-menu copies no longer come through as duplicated content.
21
+
22
+ ## [0.17.1] - 2026-09-10
23
+
24
+ ### Fixed
25
+ - sb_add and sb_import no longer store a nested node's children twice: a patch batch that gets applied more than once (as every write already is, to validate it before it lands) mutated itself on the first pass by carrying an added node by reference, so the second pass re-inserted its children into a node that already held them.
26
+ - sb_import and sb_import_site now cap a flattened row at 12 columns and treat a grid container as wrapping rather than single-line, so a page whose content wrapper is a CSS grid (a documentation site, for example) no longer imports as one row of hundreds of slivered columns.
27
+ - sb_import and sb_import_site now capture a `<pre>`/`<code>` block as one node instead of one node per syntax-highlighting `<span>`, so a code sample no longer arrives broken into dozens of single-token fragments.
28
+
9
29
  ## [0.17.0] - 2026-09-10
10
30
 
11
31
  ### Added
package/CHANGELOG.vi.md CHANGED
@@ -6,6 +6,26 @@ Mọi thay đổi đáng chú ý của dự án được ghi lại trong file n
6
6
  Định dạng dựa trên [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
7
7
  và dự án tuân theo [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
8
8
 
9
+ ## [0.18.0] - 2026-09-10
10
+
11
+ ### Added
12
+ - sb_import và sb_import_site giờ có thể mang theo một element `<video>`, một player nhúng YouTube, Vimeo, Google Map hoặc SoundCloud, và một divider `<hr>`, thay vì âm thầm bỏ qua mọi iframe vì không hỗ trợ.
13
+
14
+ ### Fixed
15
+ - sb_import và sb_import_site giờ không còn lấy nhầm header hoặc footer điều hướng của chính site nguồn khi nó nằm trong một element bọc ngoài thay vì là con trực tiếp của `<body>`, và không còn nhầm một footer chỉ được đánh dấu bằng class CSS (không có thẻ `<footer>`) là nội dung trang.
16
+ - sb_import_site giờ không còn import cùng một trang hai lần dưới hai slug khác nhau khi trang nguồn khai báo `<link rel="canonical">` trỏ tới một URL đã có trong kế hoạch, hoặc khi sitemap/crawl liệt kê các bản dịch theo ngôn ngữ của cùng một trang (`/about`, `/en/about`, `/vi/about`); ngôn ngữ của trang nhập vào được giữ lại.
17
+ - sb_import_site giờ không còn import các trang phân trang như `/blog/page/2` thành các trang riêng.
18
+ - sb_import_site giờ tuân theo các quy tắc Disallow trong robots.txt của site khi crawl hoặc đọc sitemap, trừ URL mà người gọi đã chỉ định rõ ràng.
19
+ - sb_import và sb_import_site giờ không còn nhân đôi các mục của một list lồng nhau: một `<ul>` bên trong `<li>` trước đây bị lấy một lần trong text của mục cha và lấy lại lần nữa như một mục list riêng.
20
+ - sb_import và sb_import_site giờ bỏ qua mọi element được đánh dấu `aria-hidden="true"`, để các bản sao carousel và bản sao ẩn của menu mobile không còn bị lấy thành nội dung trùng lặp.
21
+
22
+ ## [0.17.1] - 2026-09-10
23
+
24
+ ### Fixed
25
+ - sb_add và sb_import giờ không còn lưu nhân đôi các con của một node lồng nhau: một batch patch bị áp dụng nhiều hơn một lần (như mọi lần ghi vẫn vậy, để kiểm tra trước khi áp thật) trước đây tự làm thay đổi chính nó ngay ở lượt đầu vì mang theo một node vừa thêm bằng tham chiếu, khiến lượt thứ hai chèn lại các con vào một node đã sẵn có chúng.
26
+ - sb_import và sb_import_site giờ giới hạn một row được làm phẳng ở tối đa 12 cột và coi container dạng grid là tự xuống dòng thay vì nằm trên một hàng, để một trang có content wrapper là CSS grid (chẳng hạn một trang tài liệu kỹ thuật) không còn bị import thành một row hàng trăm cột mỏng dính.
27
+ - sb_import và sb_import_site giờ lấy một khối `<pre>`/`<code>` thành một node duy nhất thay vì một node cho mỗi `<span>` tô cú pháp, để một đoạn code không còn bị vỡ thành hàng chục mảnh một-token-một-node.
28
+
9
29
  ## [0.17.0] - 2026-09-10
10
30
 
11
31
  ### Added
package/dist/core/patch.js CHANGED
@@ -78,6 +78,16 @@ function parentOf(state, path) {
78
78
  * half-edited with nothing anywhere to say so — which is the exact failure shape
79
79
  * this whole repo exists to rule out.
80
80
  */
81
+ /**
82
+ * A patch's value, detached from whatever the caller still holds.
83
+ *
84
+ * Primitives are returned as they are — a style key is a string, and cloning
85
+ * one on every `sb_set` would be pure cost. Only a structure can be aliased,
86
+ * and only an alias can be mutated behind the batch's back.
87
+ */
88
+ function clone(value) {
89
+ return value !== null && typeof value === 'object' ? structuredClone(value) : value;
90
+ }
81
91
  export function applyPatches(state, patches) {
82
92
  for (const p of patches) {
83
93
  if (!isSyncablePatch(p)) {
@@ -86,7 +96,27 @@ export function applyPatches(state, patches) {
86
96
  const { holder, key } = parentOf(state, p.path);
87
97
  switch (p.op) {
88
98
  case 'set':
89
- holder[key] = p.value;
99
+ // THE VALUE IS NEVER ALIASED INTO THE DOCUMENT.
100
+ //
101
+ // `addSubtree` builds a node, emits `set nodes/<id>` carrying THAT
102
+ // object, and then emits `insert` patches that push child ids into
103
+ // `data.nodes`. Assigning the reference means the first apply MUTATES
104
+ // the patch's own value, so the batch is no longer the thing it was: a
105
+ // second apply re-establishes a node that already holds its children and
106
+ // then inserts them again.
107
+ //
108
+ // Applying a batch twice is not hypothetical — `applyAndSave` does it by
109
+ // design, once through `preview` to judge the write and once for real.
110
+ // Between v0.16.1 (which introduced that check) and this fix, EVERY
111
+ // nested `sb_add` stored each child twice, and `sb_import` three times.
112
+ // It is silent: the tree is well-formed, every id resolves, the save is
113
+ // accepted, and the page simply renders its content twice. Measured on
114
+ // a real import — 106 of 231 containers listing one child id three
115
+ // times.
116
+ //
117
+ // A copy makes the batch idempotent for this shape: the re-`set` puts a
118
+ // pristine node back, and the inserts that follow rebuild the same list.
119
+ holder[key] = clone(p.value);
90
120
  break;
91
121
  case 'unset':
92
122
  delete holder[key];
@@ -95,7 +125,7 @@ export function applyPatches(state, patches) {
95
125
  const arr = holder[key];
96
126
  if (!Array.isArray(arr))
97
127
  break;
98
- arr.splice(p.index, 0, p.value);
128
+ arr.splice(p.index, 0, clone(p.value));
99
129
  break;
100
130
  }
101
131
  case 'remove': {
package/dist/domains/site/discover.js CHANGED
@@ -39,6 +39,79 @@ const NOT_CONTENT = [
39
39
  // as often /blog/tag/x as /tag/x.
40
40
  '/tag/', '/tags/', '/author/', '/authors/',
41
41
  ];
42
+ /**
43
+ * PAGE 2 OF A LIST IS NOT A PAGE.
44
+ *
45
+ * `/blog/page/2` is the same design as `/blog` holding the next twenty records,
46
+ * and this platform renders a list from its own catalogue rather than from
47
+ * somebody else's pagination. Importing them spends the page budget on repeats
48
+ * of a layout already taken. Narrow on purpose: `/blog/2024` is a year archive
49
+ * and a real page, so only an explicit `page` segment counts.
50
+ */
51
+ const PAGINATION = /\/(?:page|pages|p)\/\d+(?:\/|$)/i;
52
+ /** A leading path segment that is a language tag: `/en`, `/vi`, `/en-us`, `/zh-hans`. */
53
+ const LOCALE_SEG = /^[a-z]{2}(?:-[a-z]{2,4})?$/i;
54
+ /** A path split into the language it is written in and the page it names. */
55
+ function localeOf(path) {
56
+ const segs = path.split('/').filter(Boolean);
57
+ if (segs.length > 0 && LOCALE_SEG.test(segs[0])) {
58
+ return { lang: segs[0].toLowerCase(), key: `/${segs.slice(1).join('/')}` };
59
+ }
60
+ return { lang: '', key: path };
61
+ }
62
+ /**
63
+ * ONE PAGE PER PAGE, not one per language.
64
+ *
65
+ * A multilingual site lists every translation in its sitemap, so `/about`,
66
+ * `/en/about` and `/vi/about` all arrive and all describe the same page. Import
67
+ * them and the merchant gets the same content three times under three slugs,
68
+ * with the page budget spent on translations of a page already taken — and this
69
+ * platform has a translations surface for exactly that job.
70
+ *
71
+ * FOLDED ONLY ON A CONFLICT. A rule that simply dropped every `/xx/` prefix
72
+ * would empty the plan for a site that serves ALL its pages under one — which
73
+ * nodejs.org does, everything under `/en`. So a group forms only when two URLs
74
+ * name the same page in different languages, which means the list can never
75
+ * come back shorter than the number of distinct pages.
76
+ *
77
+ * The entry's own language wins, because that is the one the caller pointed at.
78
+ */
79
+ function foldLocales(kept, entry, skip) {
80
+ const entryLang = localeOf(pathOf(entry)).lang;
81
+ const groups = new Map();
82
+ for (const f of kept) {
83
+ const { key } = localeOf(pathOf(f.url));
84
+ const g = groups.get(key);
85
+ if (g)
86
+ g.push(f);
87
+ else
88
+ groups.set(key, [f]);
89
+ }
90
+ const out = [];
91
+ for (const group of groups.values()) {
92
+ if (group.length === 1) {
93
+ out.push(group[0]);
94
+ continue;
95
+ }
96
+ const langs = group.map((f) => localeOf(pathOf(f.url)).lang);
97
+ if (new Set(langs).size === 1) {
98
+ // Same language, same key: not translations at all, so nothing to choose
99
+ // between. `/en/a` and `/en/a` cannot both be here — `seen` dropped that —
100
+ // so this is a key collision the fold has no opinion about.
101
+ out.push(...group);
102
+ continue;
103
+ }
104
+ const pick = group.find((f) => f.url === entry) ??
105
+ group.find((f) => localeOf(pathOf(f.url)).lang === entryLang) ??
106
+ group.find((f) => localeOf(pathOf(f.url)).lang === '') ??
107
+ group[0];
108
+ for (const f of group)
109
+ if (f !== pick)
110
+ skip('other-locale');
111
+ out.push(pick);
112
+ }
113
+ return out;
114
+ }
42
115
  /**
43
116
  * Is this path the site's plumbing rather than one of its pages?
44
117
  *
@@ -167,6 +240,75 @@ export function robotsSitemaps(txt) {
167
240
  }
168
241
  return out;
169
242
  }
243
+ /**
244
+ * The `User-agent: *` group's rules.
245
+ *
246
+ * HONOURED, not read for interest. A merchant importing their own site is one
247
+ * caller; the other points this at somebody else's server, and a tool that
248
+ * fetches a dozen pages should obey the file that exists to say which. It is
249
+ * also an accuracy win on its own terms — what a site disallows is almost always
250
+ * its plumbing, and the plumbing list here is a guess where robots.txt is the
251
+ * site's own answer.
252
+ *
253
+ * `Allow` is collected too, because the standard resolves a conflict by the
254
+ * LONGEST match and a site that disallows `/blog/` and allows `/blog/public/`
255
+ * means the second.
256
+ */
257
+ export function robotsRules(txt) {
258
+ const allow = [];
259
+ const disallow = [];
260
+ let applies = false;
261
+ for (const line of txt.split(/\r?\n/)) {
262
+ const clean = line.replace(/#.*$/, '').trim();
263
+ if (!clean)
264
+ continue;
265
+ const m = /^([A-Za-z-]+)\s*:\s*(.*)$/.exec(clean);
266
+ if (!m)
267
+ continue;
268
+ const field = m[1].toLowerCase();
269
+ const value = m[2].trim();
270
+ if (field === 'user-agent') {
271
+ applies = value === '*';
272
+ continue;
273
+ }
274
+ if (!applies || !value)
275
+ continue;
276
+ // AN EMPTY `Disallow:` MEANS ALLOW EVERYTHING, and treating it as the empty
277
+ // prefix would block every path on the site — the one parsing mistake in
278
+ // this file that turns a polite crawler into a crawler that finds nothing.
279
+ if (field === 'disallow')
280
+ disallow.push(value);
281
+ else if (field === 'allow')
282
+ allow.push(value);
283
+ }
284
+ return { allow, disallow };
285
+ }
286
+ /** Does the site ask a general crawler to leave this path alone? */
287
+ export function blockedByRobots(path, rules) {
288
+ const match = (rule) => {
289
+ // `*` and `$` are the two wildcards every major crawler honours. Anything
290
+ // else in a rule is a literal prefix.
291
+ const pattern = rule
292
+ .replace(/[.+?^${}()|[\]\\]/g, '\\$&')
293
+ .replace(/\*/g, '.*')
294
+ .replace(/\\\$$/, '$');
295
+ let re;
296
+ try {
297
+ re = new RegExp(`^${pattern}`);
298
+ }
299
+ catch {
300
+ return path.startsWith(rule) ? rule.length : -1;
301
+ }
302
+ return re.test(path) ? rule.length : -1;
303
+ };
304
+ const deny = Math.max(-1, ...rules.disallow.map(match));
305
+ if (deny < 0)
306
+ return false;
307
+ const permit = Math.max(-1, ...rules.allow.map(match));
308
+ // LONGEST MATCH WINS, and a tie goes to the crawler — that is what the
309
+ // standard says and what every implementation does.
310
+ return deny > permit;
311
+ }
170
312
  /** A slug this platform will accept, from a path. The root is the home page. */
171
313
  export function slugFor(url, taken) {
172
314
  const path = pathOf(url);
@@ -271,6 +413,16 @@ export function choosePages(entry, urls, opts = {}) {
271
413
  skip('not-content');
272
414
  continue;
273
415
  }
416
+ if (PAGINATION.test(path)) {
417
+ skip('pagination');
418
+ continue;
419
+ }
420
+ // THE ENTRY IS EXEMPT. The caller typed that URL, and a merchant whose own
421
+ // robots.txt disallows their own home page has still asked for it.
422
+ if (opts.robots && norm !== entry && blockedByRobots(path, opts.robots)) {
423
+ skip('robots-disallow');
424
+ continue;
425
+ }
274
426
  }
275
427
  if (new URL(norm).search) {
276
428
  // A query string is nearly always a filter, a sort or a page number over
@@ -291,10 +443,11 @@ export function choosePages(entry, urls, opts = {}) {
291
443
  seen.add(norm);
292
444
  kept.push({ url: norm, from: norm === entry ? 'entry' : f.from });
293
445
  }
446
+ const folded = foldLocales(kept, entry, skip);
294
447
  // GROUPS ARE COUNTED BEFORE THE CAP, because the whole point of reporting them
295
448
  // is to say what the cap is about to hide.
296
449
  const groups = {};
297
- for (const f of kept) {
450
+ for (const f of folded) {
298
451
  const seg = pathOf(f.url).split('/').filter(Boolean)[0];
299
452
  if (seg)
300
453
  groups[seg] = (groups[seg] ?? 0) + 1;
@@ -303,7 +456,7 @@ export function choosePages(entry, urls, opts = {}) {
303
456
  if (groups[k] < 3)
304
457
  delete groups[k];
305
458
  const depthOf = (u) => pathOf(u).split('/').filter(Boolean).length;
306
- kept.sort((a, b) => {
459
+ folded.sort((a, b) => {
307
460
  if (a.url === entry)
308
461
  return -1;
309
462
  if (b.url === entry)
@@ -311,11 +464,11 @@ export function choosePages(entry, urls, opts = {}) {
311
464
  const d = depthOf(a.url) - depthOf(b.url);
312
465
  return d !== 0 ? d : a.url.localeCompare(b.url);
313
466
  });
314
- const over = Math.max(0, kept.length - maxPages);
467
+ const over = Math.max(0, folded.length - maxPages);
315
468
  if (over > 0)
316
469
  skipped['over-page-limit'] = over;
317
470
  const taken = new Set();
318
- const pages = kept.slice(0, maxPages).map((f) => {
471
+ const pages = folded.slice(0, maxPages).map((f) => {
319
472
  const slug = slugFor(f.url, taken);
320
473
  return { ...f, slug, name: nameFor(slug), depth: depthOf(f.url) };
321
474
  });
@@ -342,6 +495,8 @@ export function canonFor(entry) {
342
495
  return null;
343
496
  if (isPlumbing(path))
344
497
  return null;
498
+ if (PAGINATION.test(path))
499
+ return null;
345
500
  return norm;
346
501
  };
347
502
  }
package/dist/domains/site/importmap.js CHANGED
@@ -94,6 +94,45 @@ function one(c, t) {
94
94
  },
95
95
  };
96
96
  }
97
+ case 'divider': {
98
+ return { type: 'divider', style: { width: '100%' } };
99
+ }
100
+ case 'video': {
101
+ if (!c.src)
102
+ return null;
103
+ // NO STYLE OF OUR OWN. Every media element here already seeds
104
+ // `width: 100%` + `height: fit-content` and `google-map` seeds a height
105
+ // per breakpoint; writing a literal over that detaches the node from the
106
+ // element's own responsive answer to be less correct than it.
107
+ //
108
+ // `videoRatio` is seeded 16 / 9, which is right for almost every file a
109
+ // page embeds — and markup that does not state the real ratio is not
110
+ // something to guess a crop from.
111
+ return {
112
+ type: 'video',
113
+ specials: { videoSrc: c.src, ...(c.poster ? { poster: c.poster } : {}) },
114
+ };
115
+ }
116
+ case 'embed': {
117
+ // AN EMBED IS A PROVIDER, NOT A URL. `youtube` and `vimeo` store the ID
118
+ // alone — handing them a whole watch URL renders nothing — while
119
+ // `google-map` takes the embed URL its own hint asks for and `soundcloud`
120
+ // takes a track URL under a different key again.
121
+ if (c.provider === 'youtube' || c.provider === 'vimeo') {
122
+ if (!c.videoId)
123
+ return null;
124
+ return { type: c.provider, specials: { videoId: c.videoId } };
125
+ }
126
+ if (!c.src)
127
+ return null;
128
+ if (c.provider === 'map') {
129
+ return { type: 'google-map', specials: { src: c.src, mapType: 'location' } };
130
+ }
131
+ if (c.provider === 'soundcloud') {
132
+ return { type: 'soundcloud', specials: { trackUrl: c.src } };
133
+ }
134
+ return null;
135
+ }
97
136
  case 'image': {
98
137
  if (!c.src)
99
138
  return null;
package/dist/tools/importpage.js CHANGED
@@ -5,7 +5,7 @@ import { uploadMedia } from '../transport/media.js';
5
5
  import { addSubtree } from '../domains/site/builder.js';
6
6
  import { middleEnd } from '../domains/site/traps.js';
7
7
  import { toSpecs, tokensFromPage, imageSources, rehostImages, } from '../domains/site/importmap.js';
8
- import { canonFor, choosePages, normalizeUrl, robotsSitemaps, sitemapUrls, } from '../domains/site/discover.js';
8
+ import { canonFor, choosePages, normalizeUrl, robotsRules, robotsSitemaps, sitemapUrls, } from '../domains/site/discover.js';
9
9
  import { loadSource } from '../transport/pages.js';
10
10
  import { PageDoc } from '../domains/site/document.js';
11
11
  import { request } from '../transport/http.js';
@@ -48,17 +48,9 @@ async function fetchForeign(ctx, url) {
48
48
  * One fetch and no browser, and it lists pages nothing links to — which is why
49
49
  * it is tried before the crawl rather than after it.
50
50
  */
51
- export async function fromSitemap(ctx, entry) {
51
+ export async function fromSitemap(ctx, entry, declaredIn) {
52
52
  const origin = new URL(entry).origin;
53
- const declared = new Set();
54
- // ROBOTS.TXT IS WHERE A SITE SAYS WHERE ITS SITEMAP REALLY IS, and plenty of
55
- // real ones are not at /sitemap.xml — a shop platform names
56
- // /sitemap_products_1.xml, a CMS a dated path. Guessing only the default is how
57
- // a site with a perfectly good sitemap gets crawled instead.
58
- const robots = await fetchForeign(ctx, `${origin}/robots.txt`);
59
- if (robots)
60
- for (const u of robotsSitemaps(robots))
61
- declared.add(u);
53
+ const declared = new Set(declaredIn);
62
54
  for (const guess of ['/sitemap.xml', '/sitemap_index.xml', '/sitemap-index.xml']) {
63
55
  declared.add(`${origin}${guess}`);
64
56
  }
@@ -132,20 +124,37 @@ async function existingPages(ctx, siteId) {
132
124
  * `include` or `max_pages`, not a slower way to find the same list.
133
125
  */
134
126
  async function discoverSite(ctx, entry, opts) {
135
- const listed = await fromSitemap(ctx, entry);
127
+ // ROBOTS.TXT ANSWERS TWO QUESTIONS AND IS FETCHED ONCE. Where the sitemap
128
+ // really is — plenty are not at /sitemap.xml, a shop platform names
129
+ // /sitemap_products_1.xml and a CMS a dated path — and which paths a general
130
+ // crawler is asked to leave alone.
131
+ const origin = new URL(entry).origin;
132
+ const txt = await fetchForeign(ctx, `${origin}/robots.txt`);
133
+ const robots = txt ? robotsRules(txt) : undefined;
134
+ const listed = await fromSitemap(ctx, entry, txt ? robotsSitemaps(txt) : []);
136
135
  if (listed)
137
- return { source: 'sitemap', urls: listed, titles: new Map(), visited: 0 };
136
+ return { source: 'sitemap', urls: listed, titles: new Map(), visited: 0, robots, aliases: 0 };
138
137
  const crawled = await crawlLinks(entry, {
139
138
  depth: opts.depth ?? 1,
140
139
  maxVisits: opts.maxVisits ?? 24,
141
140
  canon: canonFor(entry),
142
141
  });
143
- return {
144
- source: 'links',
145
- urls: crawled.urls.map((u) => ({ url: u, from: u === entry ? 'entry' : 'links' })),
146
- titles: crawled.titles,
147
- visited: crawled.visited,
148
- };
142
+ // A PAGE THAT NAMES ANOTHER ADDRESS AS ITS OWN IS THAT PAGE. Folded here, on
143
+ // the crawl path, where the answer is already in hand — the sitemap path has
144
+ // no canonical until the page is opened, and the import pass folds that one.
145
+ const seen = new Set();
146
+ const urls = [];
147
+ let aliases = 0;
148
+ for (const u of crawled.urls) {
149
+ const real = crawled.canonical.get(u) ?? u;
150
+ if (real !== u)
151
+ aliases += 1;
152
+ if (seen.has(real))
153
+ continue;
154
+ seen.add(real);
155
+ urls.push({ url: real, from: real === entry ? 'entry' : 'links' });
156
+ }
157
+ return { source: 'links', urls, titles: crawled.titles, visited: crawled.visited, robots, aliases };
149
158
  }
150
159
  export function registerImportTools(server, ctx, session) {
151
160
  server.registerTool('sb_import', {
@@ -321,6 +330,7 @@ export function registerImportTools(server, ctx, session) {
321
330
  maxPages: max_pages,
322
331
  include,
323
332
  exclude,
333
+ robots: found.robots,
324
334
  });
325
335
  // A CRAWL ALREADY READ THE TITLE. `nameFor` derives a name from the slug
326
336
  // because a sitemap offers nothing else, but the link crawl opened every
@@ -394,7 +404,9 @@ export function registerImportTools(server, ctx, session) {
394
404
  discovered_by: found.source,
395
405
  ...(found.visited ? { pages_read_to_find_them: found.visited } : {}),
396
406
  pages: plan.pages.map((p) => ({ url: p.url, slug: p.slug, name: p.name, ...lands(p) })),
397
- ...(Object.keys(plan.skipped).length ? { skipped: plan.skipped } : {}),
407
+ ...(Object.keys(plan.skipped).length || found.aliases
408
+ ? { skipped: { ...plan.skipped, ...(found.aliases ? { 'canonical-alias': found.aliases } : {}) } }
409
+ : {}),
398
410
  ...(unlistable
399
411
  ? {
400
412
  landing_unknown: `This site's own pages could not be read (${unlistable}), so which of the above ` +
@@ -467,6 +479,13 @@ export function registerImportTools(server, ctx, session) {
467
479
  // three pages built, nine not, and no report saying which.
468
480
  const built = [];
469
481
  const failed = [];
482
+ // WHAT EACH PAGE SAYS ITS OWN ADDRESS IS. A sitemap cannot tell you that
483
+ // two of its entries are one page — only the page can, and only once it is
484
+ // open. Measured: modelcontextprotocol.io's home page declares a dated
485
+ // docs path as its canonical, so `/` and that path are the same content
486
+ // under two slugs, and nothing in the plan looks wrong.
487
+ const identities = new Set();
488
+ const aliased = [];
470
489
  let lastOpened = '';
471
490
  for (const p of plan.pages) {
472
491
  const shot = byUrl.get(p.url);
@@ -474,6 +493,12 @@ export function registerImportTools(server, ctx, session) {
474
493
  failed.push({ url: p.url, why: shot ? shot.why : 'was not read' });
475
494
  continue;
476
495
  }
496
+ const identity = normalizeUrl(shot.result.canonical ?? p.url) ?? p.url;
497
+ if (identities.has(identity)) {
498
+ aliased.push({ url: p.url, same_as: identity });
499
+ continue;
500
+ }
501
+ identities.add(identity);
477
502
  try {
478
503
  const sections = rehosted.size > 0 ? rehostImages(shot.result.sections, rehosted) : shot.result.sections;
479
504
  const specs = toSpecs(sections, tokens);
@@ -557,7 +582,10 @@ export function registerImportTools(server, ctx, session) {
557
582
  discovered_by: found.source,
558
583
  built,
559
584
  ...(failed.length ? { failed } : {}),
560
- ...(Object.keys(plan.skipped).length ? { skipped: plan.skipped } : {}),
585
+ ...(aliased.length ? { same_page: aliased } : {}),
586
+ ...(Object.keys(plan.skipped).length || found.aliases
587
+ ? { skipped: { ...plan.skipped, ...(found.aliases ? { 'canonical-alias': found.aliases } : {}) } }
588
+ : {}),
561
589
  images: {
562
590
  copied: rehosted.size,
563
591
  ...(failedImages.length ? { failed: failedImages } : {}),
package/dist/vision/capture.js CHANGED
@@ -63,10 +63,31 @@ function capturePage(limits) {
63
63
  const bordered = cs.borderStyle !== '' && cs.borderStyle !== 'none' && parseFloat(cs.borderWidth || '0') > 0;
64
64
  return cs.display !== 'inline' && (filled || bordered);
65
65
  };
66
+ // IFRAME IS NOT HERE ANY MORE. It was, and it took every embedded video, every
67
+ // map and every audio player with it — silently, as a skip count. The platform
68
+ // has `video`, `youtube`, `vimeo`, `soundcloud` and `google-map`; a hero video
69
+ // and a contact page's map are ordinary things to import, and they were the
70
+ // one kind of content that could not survive the trip at all.
66
71
  const IGNORE = new Set([
67
- 'SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'SVG', 'PATH', 'IFRAME', 'CANVAS',
72
+ 'SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'SVG', 'PATH', 'CANVAS',
68
73
  'NAV', 'FORM', 'INPUT', 'SELECT', 'TEXTAREA', 'BUTTON',
69
74
  ]);
75
+ /** The provider and id behind an embed URL, or null if this platform has no element for it. */
76
+ const embedOf = (raw) => {
77
+ const u = raw.split('?')[0];
78
+ let m = /(?:youtube(?:-nocookie)?\.com\/(?:embed|v|shorts)\/|youtu\.be\/)([A-Za-z0-9_-]{6,})/.exec(u);
79
+ if (m)
80
+ return { provider: 'youtube', videoId: m[1] };
81
+ m = /player\.vimeo\.com\/video\/(\d+)/.exec(u);
82
+ if (m)
83
+ return { provider: 'vimeo', videoId: m[1] };
84
+ if (/(?:google\.[a-z.]+|maps\.google\.[a-z.]+)\/maps?\/embed/.test(u)) {
85
+ return { provider: 'map', src: raw };
86
+ }
87
+ if (/w\.soundcloud\.com\/player/.test(u))
88
+ return { provider: 'soundcloud', src: raw };
89
+ return null;
90
+ };
70
91
  /**
71
92
  * The renderable content under one section, AS A TREE.
72
93
  *
@@ -83,6 +104,77 @@ function capturePage(limits) {
83
104
  * merely wraps is not a design decision and reproducing it would nest the
84
105
  * result ten deep for nothing.
85
106
  */
107
+ // THE SOURCE'S OWN HEADER AND FOOTER ARE NEVER WANTED. The target site has its
108
+ // own, as shared globals, and importing somebody else's navigation onto a
109
+ // storefront is a second menu pointing at a different website.
110
+ //
111
+ // NOT NAMED `chrome`: that is a BROWSER GLOBAL (the extension API), so a
112
+ // `const chrome` declared in a nested scope left every OTHER scope resolving
113
+ // the name to `window.chrome` — `chrome.has is not a function`, thrown inside
114
+ // `evaluate`, which kills the whole capture. The same shape as the closure
115
+ // trap this file already carries, reached from the opposite direction.
116
+ //
117
+ // PAGE-LEVEL, BY THE SPEC'S OWN DEFINITION rather than by depth. This used to
118
+ // ask whether the element was a DIRECT child of `<body>`, which almost no real
119
+ // site satisfies — one wrapper div is enough — so blender.org's footer came
120
+ // through as eleven sections of link columns and the page a merchant asked for
121
+ // was its site map. `<header>` and `<footer>` belong to their nearest
122
+ // SECTIONING ancestor (article, aside, nav, section), so one with none of
123
+ // those above it is the page's, however deeply it is wrapped; one inside an
124
+ // `<article>` is that article's byline, and one inside a `<section>` is the
125
+ // hero the old comment was right to protect. `<main>` is not sectioning
126
+ // content, so it does not shield a footer.
127
+ const sectioning = Array.from(document.querySelectorAll('article, section, aside, nav'));
128
+ // A CLASS NAME IS EVIDENCE FOR A FOOTER AND NOT FOR A HEADER, and the
129
+ // asymmetry is the whole point. blender.org marks its site map
130
+ // `<div class="footer-navigation">` — no `<footer>` tag anywhere near it — so
131
+ // the spec rule alone let eleven sections of somebody else's links through as
132
+ // the page a merchant asked for. The same trick on the header side would eat
133
+ // HEROES: blender's own first band is `<div class="hero header-size-large">`,
134
+ // and losing the first thing on a landing page costs more than a stray footer.
135
+ // A header is caught by its tag or its ARIA role, and its links are `<nav>`,
136
+ // which is ignored already.
137
+ //
138
+ // The token must START with `footer` (footer, footer-note, footer__inner) so
139
+ // `card-footer` inside an ordinary div is not swept up with it.
140
+ const footerish = (el) => {
141
+ const words = `${el.getAttribute('id') ?? ''} ${String(el.className ?? '')}`.toLowerCase();
142
+ for (const w of words.split(/[\s]+/)) {
143
+ if (!w)
144
+ continue;
145
+ if (w === 'colophon' || w === 'site-footer' || w === 'page-footer')
146
+ return true;
147
+ if (w === 'footer' || w.indexOf('footer-') === 0 || w.indexOf('footer_') === 0)
148
+ return true;
149
+ }
150
+ return false;
151
+ };
152
+ const pageChromeRoots = [];
153
+ const chromeCandidates = Array.from(document.querySelectorAll('header, footer, [role="banner"], [role="contentinfo"], [class*="footer"], [id*="footer"]'));
154
+ for (const el of chromeCandidates) {
155
+ const role = el.getAttribute('role');
156
+ const isChrome = el.tagName === 'HEADER' ||
157
+ el.tagName === 'FOOTER' ||
158
+ role === 'banner' ||
159
+ role === 'contentinfo' ||
160
+ footerish(el);
161
+ if (!isChrome)
162
+ continue;
163
+ if (sectioning.some((sec) => sec !== el && sec.contains(el)))
164
+ continue;
165
+ // A NESTED ONE ADDS NOTHING: the outer root already covers it, and keeping
166
+ // both makes the containment test scan the same subtree twice.
167
+ if (pageChromeRoots.some((c) => c.contains(el)))
168
+ continue;
169
+ pageChromeRoots.push(el);
170
+ }
171
+ // INSIDE the chrome, not equal to it. A real footer holds `<section>`s, and
172
+ // the candidate walk deliberately takes the INNERMOST sections — so on
173
+ // blender.org the candidates were the footer's own link columns, none of which
174
+ // IS the footer, and eleven sections of somebody else's site map came through
175
+ // as the page. Asking about containment is the same question the candidate
176
+ // list already answers for nesting.
177
+ const inPageChrome = (el) => pageChromeRoots.some((c) => c === el || c.contains(el));
86
178
  const leaves = (root) => {
87
179
  const walkChildren = (el) => {
88
180
  const kids = [];
@@ -104,7 +196,24 @@ function capturePage(limits) {
104
196
  skip('over-node-limit');
105
197
  return [];
106
198
  }
199
+ if (inPageChrome(el)) {
200
+ skip('page-chrome');
201
+ return [];
202
+ }
107
203
  const tag = el.tagName;
204
+ // WHAT THE PAGE ITSELF SAYS IS NOT CONTENT.
205
+ //
206
+ // `aria-hidden="true"` is the author's own mark for decoration and for
207
+ // duplicates — a carousel's cloned slides, the mobile copy of a menu that
208
+ // the desktop layout also carries, an icon that repeats the label beside
209
+ // it. Measured on real pages before this: 53 such elements on one, 15 on
210
+ // another, every one of them walked and some of them captured twice.
211
+ // Nothing here reads the accessibility tree, so this attribute is the only
212
+ // place that answer exists.
213
+ if (el.getAttribute('aria-hidden') === 'true') {
214
+ skip('aria-hidden');
215
+ return [];
216
+ }
108
217
  if (IGNORE.has(tag)) {
109
218
  skip(tag.toLowerCase());
110
219
  return [];
@@ -113,6 +222,40 @@ function capturePage(limits) {
113
222
  skip('hidden');
114
223
  return [];
115
224
  }
225
+ // A RULE BETWEEN SECTIONS IS A DESIGN DECISION, and it is one node.
226
+ if (tag === 'HR') {
227
+ taken.nodes++;
228
+ return [{ kind: 'divider' }];
229
+ }
230
+ if (tag === 'VIDEO') {
231
+ const direct = el.getAttribute('src');
232
+ const source = Array.from(el.querySelectorAll('source'))[0];
233
+ const src = direct || (source ? source.getAttribute('src') : null);
234
+ if (!src) {
235
+ skip('video-without-src');
236
+ return [];
237
+ }
238
+ taken.nodes++;
239
+ const poster = el.getAttribute('poster');
240
+ return [{
241
+ kind: 'video',
242
+ src: abs(src),
243
+ ...(poster ? { poster: abs(poster) } : {}),
244
+ }];
245
+ }
246
+ if (tag === 'IFRAME') {
247
+ const src = el.getAttribute('src');
248
+ const embed = src ? embedOf(abs(src)) : null;
249
+ if (!embed) {
250
+ // An advert, a tracking pixel, a chat widget, a comment system: real
251
+ // pages carry several, and this platform has an element for none of
252
+ // them. Counted rather than guessed at.
253
+ skip('iframe');
254
+ return [];
255
+ }
256
+ taken.nodes++;
257
+ return [{ kind: 'embed', ...embed }];
258
+ }
116
259
  if (HEADINGS.has(tag)) {
117
260
  const text = clean(el.textContent);
118
261
  if (!text)
@@ -175,14 +318,63 @@ function capturePage(limits) {
175
318
  return [];
176
319
  }
177
320
  if (tag === 'UL' || tag === 'OL') {
178
- const items = Array.from(el.querySelectorAll('li'))
179
- .map((li) => clean(li.textContent))
180
- .filter(Boolean);
321
+ // A NESTED LIST WAS TAKEN TWICE, and the duplication reads as a page
322
+ // that stutters. `querySelectorAll('li')` returns the nested items as
323
+ // well as the outer ones, and an outer item's `textContent` ALREADY
324
+ // contains its sublist — so every nested entry arrived once inside its
325
+ // parent's line and once again on its own. Measured on a real import:
326
+ // one section of blender.org repeated five sublists that way.
327
+ //
328
+ // Walked by DIRECT children instead, with each item's own words
329
+ // separated from its sublist's and the sublist flattened after it. This
330
+ // platform's `list` is flat, so flattening is the honest translation —
331
+ // and the order a reader sees is preserved.
332
+ const items = [];
333
+ const collect = (list) => {
334
+ for (const li of Array.from(list.children)) {
335
+ if (li.tagName !== 'LI')
336
+ continue;
337
+ const sublists = Array.from(li.children).filter((c) => c.tagName === 'UL' || c.tagName === 'OL');
338
+ let text = clean(li.textContent);
339
+ for (const sub of sublists) {
340
+ const inner = clean(sub.textContent);
341
+ // textContent runs in document order, so a sublist's words are the
342
+ // tail of its parent's. Only strip what is actually there.
343
+ if (inner && text.length > inner.length && text.slice(-inner.length) === inner) {
344
+ text = clean(text.slice(0, text.length - inner.length));
345
+ }
346
+ }
347
+ if (text)
348
+ items.push(text);
349
+ for (const sub of sublists)
350
+ collect(sub);
351
+ }
352
+ };
353
+ collect(el);
181
354
  if (!items.length)
182
355
  return [];
183
356
  taken.nodes++;
184
357
  return [{ kind: 'list', items }];
185
358
  }
359
+ // A CODE BLOCK IS ONE THING, and walking into it produces rubble.
360
+ //
361
+ // Every syntax highlighter wraps each token in its own <span>, so the leaf
362
+ // walk took them one at a time: a twenty-line JSON config arrived as forty
363
+ // separate text nodes — `{`, `"mcpServers"`, `: {` — each its own block on
364
+ // its own line. Measured on a real import; it also ate forty of the node
365
+ // budget to say what one node says.
366
+ //
367
+ // Whitespace is collapsed like any other text because this platform has no
368
+ // code element to preserve it in — the six kinds are what can cross — so
369
+ // the honest translation is one paragraph the merchant can then restyle,
370
+ // not a shredded imitation of a listing.
371
+ if (tag === 'PRE' || tag === 'CODE') {
372
+ const text = clean(el.textContent);
373
+ if (!text)
374
+ return [];
375
+ taken.nodes++;
376
+ return [{ kind: 'text', text }];
377
+ }
186
378
  if (tag === 'P' || tag === 'BLOCKQUOTE') {
187
379
  const text = clean(el.textContent);
188
380
  if (!text)
@@ -197,12 +389,34 @@ function capturePage(limits) {
197
389
  cs.display === 'inline-flex' || cs.display === 'inline-grid';
198
390
  // A ROW is worth keeping; a column is what the page already is, so
199
391
  // wrapping one in a group would add a level that renders identically.
200
- const row = cs.display.indexOf('grid') >= 0
201
- ? true
202
- : cs.flexDirection === 'row' || cs.flexDirection === 'row-reverse';
203
- if (lays && row && kids.length >= 2) {
392
+ const grid = cs.display.indexOf('grid') >= 0;
393
+ const row = grid || cs.flexDirection === 'row' || cs.flexDirection === 'row-reverse';
394
+ // A ROW OF 279 IS NOT A ROW.
395
+ //
396
+ // Measured on a real import of a documentation site: its content wrapper
397
+ // is a GRID, every grid was read as a single row, and the whole page came
398
+ // back as one flex row of 279 columns — each column a sliver, every line
399
+ // of prose broken to one word, and 80 nodes hanging past the viewport.
400
+ // The page was structurally perfect and visually destroyed.
401
+ //
402
+ // A design row is a feature trio, a card shelf, a logo wall: a handful of
403
+ // columns, chosen. Past that the container is not arranging things side
404
+ // by side, it is the page's own content column and the browser is
405
+ // wrapping it — so the honest translation is the stack it already reads
406
+ // as. Twelve is above any real row seen here and far below a content
407
+ // grid.
408
+ const ROW_MAX = 12;
409
+ if (lays && row && kids.length >= 2 && kids.length <= ROW_MAX) {
204
410
  taken.nodes++;
205
- return [{ kind: 'group', direction: 'row', wrap: cs.flexWrap === 'wrap', children: kids }];
411
+ return [{
412
+ kind: 'group',
413
+ direction: 'row',
414
+ // A GRID ALWAYS WRAPS — that is what a grid IS — and `flexWrap` reads
415
+ // `nowrap` on one because the property does not apply. Carrying that
416
+ // literally gave the columns nowhere to go at any width.
417
+ wrap: grid || cs.flexWrap === 'wrap',
418
+ children: kids,
419
+ }];
206
420
  }
207
421
  return kids;
208
422
  }
@@ -245,20 +459,10 @@ function capturePage(limits) {
245
459
  // document is one candidate too, and keeping THAT would reduce every page to a
246
460
  // single band. The finest ones are the page's actual bands.
247
461
  candidates = candidates.filter((el) => !candidates.some((o) => o !== el && el.contains(o)));
248
- // THE SOURCE'S OWN HEADER AND FOOTER ARE NEVER WANTED. The target site has its
249
- // own, as shared globals, and importing somebody else's navigation onto a
250
- // storefront is a second menu pointing at a different website. Only the
251
- // PAGE-LEVEL ones are dropped — a `<header>` inside a section is a hero, and
252
- // excluding those would lose the first thing on most landing pages.
253
- const chrome = new Set();
254
- for (const el of Array.from(document.body.children)) {
255
- if (el.tagName === 'HEADER' || el.tagName === 'FOOTER')
256
- chrome.add(el);
257
- }
258
462
  const build = (from) => {
259
463
  const acc = [];
260
464
  for (const el of from) {
261
- if (chrome.has(el)) {
465
+ if (inPageChrome(el)) {
262
466
  skip('page-chrome');
263
467
  continue;
264
468
  }
@@ -299,7 +503,15 @@ function capturePage(limits) {
299
503
  const main = document.querySelectorAll('main')[0] ?? document.body;
300
504
  sections = build(Array.from(main.children));
301
505
  }
302
- return { url: here, title: clean(document.title), sections, skipped };
506
+ const link = Array.from(document.querySelectorAll('link[rel="canonical"]'))[0];
507
+ const canonical = link ? (link.getAttribute('href') ?? '') : '';
508
+ return {
509
+ url: here,
510
+ title: clean(document.title),
511
+ ...(canonical ? { canonical: abs(canonical) } : {}),
512
+ sections,
513
+ skipped,
514
+ };
303
515
  }
304
516
  /**
305
517
  * ONE BROWSER FOR THE WHOLE CALL.
@@ -421,7 +633,20 @@ function linksOnPage() {
421
633
  // One unresolvable href must not kill the crawl.
422
634
  }
423
635
  }
424
- return { title: document.title, links };
636
+ const link = Array.from(document.querySelectorAll('link[rel="canonical"]'))[0];
637
+ let canonical = '';
638
+ if (link) {
639
+ const href = link.getAttribute('href');
640
+ if (href) {
641
+ try {
642
+ canonical = new URL(href, here).href;
643
+ }
644
+ catch {
645
+ canonical = '';
646
+ }
647
+ }
648
+ }
649
+ return { title: document.title, canonical, links };
425
650
  }
426
651
  /**
427
652
  * Walk a site's own links from one entry page, breadth first.
@@ -438,6 +663,7 @@ export async function crawlLinks(entry, opts) {
438
663
  const maxVisits = opts.maxVisits ?? 24;
439
664
  const found = new Set([entry]);
440
665
  const titles = new Map();
666
+ const canonical = new Map();
441
667
  let visited = 0;
442
668
  await withBrowser(async (browser) => {
443
669
  let frontier = [entry];
@@ -455,6 +681,14 @@ export async function crawlLinks(entry, opts) {
455
681
  const got = await readPage(browser, url, 1440, (page) => page.evaluate(linksOnPage));
456
682
  if (got.title)
457
683
  titles.set(url, got.title);
684
+ if (got.canonical) {
685
+ const c = opts.canon(got.canonical);
686
+ // A PAGE THAT NAMES ANOTHER ADDRESS AS ITS OWN is that page. Recorded
687
+ // rather than acted on here: the crawl still walks this copy for its
688
+ // links, and the caller folds the two when it builds the plan.
689
+ if (c && c !== url)
690
+ canonical.set(url, c);
691
+ }
458
692
  // A LEVEL BELOW THE LAST IS WALKED FOR ITS LINKS AND NOT QUEUED: at
459
693
  // `depth` the crawl still wants what that page points at, it just
460
694
  // must not navigate any further.
@@ -473,5 +707,5 @@ export async function crawlLinks(entry, opts) {
473
707
  frontier = next;
474
708
  }
475
709
  });
476
- return { urls: [...found], titles, visited };
710
+ return { urls: [...found], titles, canonical, visited };
477
711
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "sbuilder-mcp",
3
- "version": "0.17.0",
3
+ "version": "0.18.0",
4
4
  "description": "MCP server that designs and operates a Store Builder site — pages, data, theme and publish — through the platform's own API and live-edit protocol.",
5
5
  "mcpName": "io.github.vuluu2k/sbuilder-mcp",
6
6
  "type": "module",