shapewall 0.0.0-stage → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (304) hide show
  1. package/CHANGELOG.md +139 -0
  2. package/Dockerfile.agent +45 -0
  3. package/LICENSE +202 -0
  4. package/NOTICE +12 -0
  5. package/README.md +651 -2
  6. package/THIRD_PARTY.md +165 -0
  7. package/bin/shapewall.js +29 -0
  8. package/dist/agent/package-lock.json +8384 -0
  9. package/dist/client/assets/BoardChat-D5SFd8HE.js +28 -0
  10. package/dist/client/assets/BuzzDialog-HWXBy52W.js +1 -0
  11. package/dist/client/assets/ChecksPanel-DTdImWoD.js +1 -0
  12. package/dist/client/assets/History-CEv6b-av.js +1 -0
  13. package/dist/client/assets/LocalSettings-C4vriEVQ.js +1 -0
  14. package/dist/client/assets/Management-DCfhcEUY.js +1 -0
  15. package/dist/client/assets/QuickConnect-DQGgUsl1.js +1 -0
  16. package/dist/client/assets/ShareDialog-C_plIto0.js +2 -0
  17. package/dist/client/assets/app-update-dialog-2hPz_N9G.js +1 -0
  18. package/dist/client/assets/circle-x-BZMPi8Zf.js +1 -0
  19. package/dist/client/assets/index-ByQRKVit.css +2 -0
  20. package/dist/client/assets/index-DweQ8Qtu.js +29 -0
  21. package/dist/client/assets/refresh-cw-CRow0Nbq.js +1 -0
  22. package/dist/client/assets/runtime-Bft6x_6E.js +4 -0
  23. package/dist/client/assets/square-A8EnIYLR.js +1 -0
  24. package/dist/client/assets/tts-dist-DYlBQmwG.js +18 -0
  25. package/dist/client/assets/tts-en-g2p-BPUtBFGG.js +1 -0
  26. package/dist/client/assets/tts-ort-wasm-simd-threaded-qh98TCA_.wasm +0 -0
  27. package/dist/client/assets/tts-ort.wasm.bundle.min-rEkcs_ow.js +12 -0
  28. package/dist/client/assets/tts-rolldown-runtime-D8NudN84.js +1 -0
  29. package/dist/client/assets/tts-spacy-en-tokenizer-6SNMl--A.json +1 -0
  30. package/dist/client/assets/tts-tts.worker-Bzm6aWbv.js +1 -0
  31. package/dist/client/assets/tts-us_gold-DcjeNMH1.json +92589 -0
  32. package/dist/client/assets/tts-us_silver-C964mEB5.json +93363 -0
  33. package/dist/client/assets/ui-CLbws0rx.js +41 -0
  34. package/dist/client/build-info.json +1 -0
  35. package/dist/client/favicon.svg +7 -0
  36. package/dist/client/icons/apple-touch-icon.png +0 -0
  37. package/dist/client/icons/icon-192.png +0 -0
  38. package/dist/client/icons/icon-512.png +0 -0
  39. package/dist/client/icons/maskable-512.png +0 -0
  40. package/dist/client/index.html +25 -0
  41. package/dist/client/manifest.webmanifest +15 -0
  42. package/dist/client/sw.js +53 -0
  43. package/dist/client/theme-boot.js +27 -0
  44. package/dist/console/assets/LocalAccountsApp-BQCcmj3x.js +1 -0
  45. package/dist/console/assets/LocalConsole-Bb2FUDs7.js +1 -0
  46. package/dist/console/assets/app-update-dialog-DIeYepLZ.js +1 -0
  47. package/dist/console/assets/console-C7uJLLOn.js +51 -0
  48. package/dist/console/assets/console-R8IBY3Am.css +2 -0
  49. package/dist/console/build-info.json +1 -0
  50. package/dist/console/console.html +21 -0
  51. package/dist/console/favicon.svg +7 -0
  52. package/dist/console/icons/apple-touch-icon.png +0 -0
  53. package/dist/console/icons/icon-192.png +0 -0
  54. package/dist/console/icons/icon-512.png +0 -0
  55. package/dist/console/icons/maskable-512.png +0 -0
  56. package/dist/console/manifest.webmanifest +15 -0
  57. package/dist/console/theme-boot.js +27 -0
  58. package/docs/agent-hardening.md +342 -0
  59. package/docs/agent-recipes.md +187 -0
  60. package/docs/agent-workflow.md +320 -0
  61. package/docs/agents.md +894 -0
  62. package/docs/architecture.md +242 -0
  63. package/docs/buzz.md +140 -0
  64. package/docs/capture-warnings.md +57 -0
  65. package/docs/config.md +489 -0
  66. package/docs/deployment.md +289 -0
  67. package/docs/design-system.md +162 -0
  68. package/docs/github.md +107 -0
  69. package/docs/hosted.md +764 -0
  70. package/docs/images/board-overview.png +0 -0
  71. package/docs/images/journeys.png +0 -0
  72. package/docs/images/live-comment.png +0 -0
  73. package/docs/images/phone-comment.png +0 -0
  74. package/docs/images/phone-overview.png +0 -0
  75. package/docs/images/variant.png +0 -0
  76. package/docs/interaction-audits.md +40 -0
  77. package/docs/journeys.md +137 -0
  78. package/docs/local-accounts.md +80 -0
  79. package/docs/notifications.md +237 -0
  80. package/docs/page-checks.md +69 -0
  81. package/docs/performance.md +174 -0
  82. package/docs/starter-agent.md +325 -0
  83. package/docs/updates.md +87 -0
  84. package/examples/audits/interaction.audit.json +17 -0
  85. package/package.json +132 -4
  86. package/src/accounts/avatar.js +121 -0
  87. package/src/accounts/local.js +464 -0
  88. package/src/agent/acp/command.js +204 -0
  89. package/src/agent/acp/detect.js +75 -0
  90. package/src/agent/acp/sandbox.js +160 -0
  91. package/src/agent/agent-health.js +75 -0
  92. package/src/agent/backends/acp.js +1297 -0
  93. package/src/agent/backends/index.js +61 -0
  94. package/src/agent/backends/openai.js +308 -0
  95. package/src/agent/composer.js +221 -0
  96. package/src/agent/config.js +338 -0
  97. package/src/agent/context.js +191 -0
  98. package/src/agent/control.js +122 -0
  99. package/src/agent/errors.js +17 -0
  100. package/src/agent/feedback-loop.js +137 -0
  101. package/src/agent/manage.js +94 -0
  102. package/src/agent/next-steps.js +51 -0
  103. package/src/agent/owner-mcp.js +38 -0
  104. package/src/agent/plain-text.js +52 -0
  105. package/src/agent/ratings-core.js +213 -0
  106. package/src/agent/ratings.js +100 -0
  107. package/src/agent/reply-text.js +67 -0
  108. package/src/agent/routes.js +1065 -0
  109. package/src/agent/run.js +326 -0
  110. package/src/agent/settings-routes.js +704 -0
  111. package/src/agent/showable.js +23 -0
  112. package/src/agent/starter/approvals.js +181 -0
  113. package/src/agent/starter/build-image.js +90 -0
  114. package/src/agent/starter/cli.js +163 -0
  115. package/src/agent/starter/disk.js +53 -0
  116. package/src/agent/starter/doctor.js +180 -0
  117. package/src/agent/starter/egress-proxy.js +351 -0
  118. package/src/agent/starter/gate.js +90 -0
  119. package/src/agent/starter/loop.js +114 -0
  120. package/src/agent/starter/model-relay.js +289 -0
  121. package/src/agent/starter/pause.js +14 -0
  122. package/src/agent/starter/probe.js +272 -0
  123. package/src/agent/starter/prompt.js +41 -0
  124. package/src/agent/starter/push-guard.js +146 -0
  125. package/src/agent/starter/rpc.js +164 -0
  126. package/src/agent/starter/runtime.js +287 -0
  127. package/src/agent/starter/sandbox-main.js +121 -0
  128. package/src/agent/starter/settings.js +290 -0
  129. package/src/agent/starter/source.js +69 -0
  130. package/src/agent/starter/supervisor.js +755 -0
  131. package/src/agent/starter/sweep.js +46 -0
  132. package/src/agent/starter/tools.js +524 -0
  133. package/src/agent/starter/watch.js +84 -0
  134. package/src/agent/starter/worktree.js +409 -0
  135. package/src/agent/text-diff.js +210 -0
  136. package/src/agent/thread-history.js +223 -0
  137. package/src/agent/threads.js +399 -0
  138. package/src/agent/tools.js +200 -0
  139. package/src/agent/turn-changes.js +424 -0
  140. package/src/agent/undo-state.js +26 -0
  141. package/src/agent/user-settings.js +308 -0
  142. package/src/auth/record.js +375 -0
  143. package/src/broken-links.js +164 -0
  144. package/src/buzz/client.js +75 -0
  145. package/src/buzz/config.js +151 -0
  146. package/src/buzz/environment.js +13 -0
  147. package/src/buzz/gateway.js +192 -0
  148. package/src/buzz/routes.js +61 -0
  149. package/src/buzz/signer-child.js +205 -0
  150. package/src/buzz/store.js +89 -0
  151. package/src/buzz/transport.js +60 -0
  152. package/src/buzz/verify.js +140 -0
  153. package/src/capture/actions.js +159 -0
  154. package/src/capture/capture.js +776 -0
  155. package/src/capture/dom-names.js +189 -0
  156. package/src/capture/element-map.js +509 -0
  157. package/src/capture/live.js +827 -0
  158. package/src/capture/media-scheme.js +317 -0
  159. package/src/capture/network-summary.js +86 -0
  160. package/src/capture/pacing.js +34 -0
  161. package/src/capture/requests.js +167 -0
  162. package/src/capture/sanitize.js +752 -0
  163. package/src/checks/accessibility.js +134 -0
  164. package/src/checks/budgets.js +58 -0
  165. package/src/checks/cli.js +55 -0
  166. package/src/checks/comparison.js +113 -0
  167. package/src/checks/environments.js +32 -0
  168. package/src/checks/flow-config.js +82 -0
  169. package/src/checks/flow-origin.js +53 -0
  170. package/src/checks/flows.js +398 -0
  171. package/src/checks/lighthouse-auth.js +145 -0
  172. package/src/checks/lighthouse-findings.js +130 -0
  173. package/src/checks/lighthouse.js +395 -0
  174. package/src/checks/robots.js +85 -0
  175. package/src/checks/routes.js +52 -0
  176. package/src/checks/run-control.js +259 -0
  177. package/src/checks/runner.js +724 -0
  178. package/src/checks/schema.js +133 -0
  179. package/src/checks/seo-analysis.js +443 -0
  180. package/src/checks/seo-probe.js +190 -0
  181. package/src/checks/seo-snapshot.js +99 -0
  182. package/src/checks/store.js +214 -0
  183. package/src/checks/transport.js +47 -0
  184. package/src/cli/args.js +319 -0
  185. package/src/cli/capture.js +79 -0
  186. package/src/cli/commands.js +92 -0
  187. package/src/cli/comments.js +84 -0
  188. package/src/cli/help.js +243 -0
  189. package/src/cli/init.js +214 -0
  190. package/src/cli/misc.js +64 -0
  191. package/src/cli/notify.js +87 -0
  192. package/src/cli/open.js +22 -0
  193. package/src/cli/serve.js +274 -0
  194. package/src/cli/variant.js +82 -0
  195. package/src/cli/watch.js +70 -0
  196. package/src/cli.js +130 -0
  197. package/src/comment-fields.js +76 -0
  198. package/src/comments-cli.js +208 -0
  199. package/src/config-reload.js +100 -0
  200. package/src/config.js +1061 -0
  201. package/src/devices-cli.js +21 -0
  202. package/src/diff.js +95 -0
  203. package/src/discover.js +284 -0
  204. package/src/doctor.js +295 -0
  205. package/src/event-log.js +268 -0
  206. package/src/events-cli.js +102 -0
  207. package/src/events.js +113 -0
  208. package/src/export.js +119 -0
  209. package/src/feedback.js +246 -0
  210. package/src/github.js +551 -0
  211. package/src/hosted/LICENSE.md +105 -0
  212. package/src/hosted/access-cli.js +78 -0
  213. package/src/hosted/admin.js +682 -0
  214. package/src/hosted/agent-approvals.js +163 -0
  215. package/src/hosted/agent-policy.js +189 -0
  216. package/src/hosted/app.js +510 -0
  217. package/src/hosted/auth-routes.js +540 -0
  218. package/src/hosted/auth.js +536 -0
  219. package/src/hosted/backup.js +483 -0
  220. package/src/hosted/board-context.js +35 -0
  221. package/src/hosted/board-routes.js +486 -0
  222. package/src/hosted/board-view.js +94 -0
  223. package/src/hosted/clean-worker.js +24 -0
  224. package/src/hosted/cleaner.js +89 -0
  225. package/src/hosted/comments.js +355 -0
  226. package/src/hosted/console-routes.js +401 -0
  227. package/src/hosted/cookies.js +18 -0
  228. package/src/hosted/crypto.js +80 -0
  229. package/src/hosted/db.js +496 -0
  230. package/src/hosted/device-routes.js +175 -0
  231. package/src/hosted/device.js +233 -0
  232. package/src/hosted/events.js +63 -0
  233. package/src/hosted/guards.js +172 -0
  234. package/src/hosted/guest-view.js +65 -0
  235. package/src/hosted/http.js +39 -0
  236. package/src/hosted/import.js +300 -0
  237. package/src/hosted/mail.js +73 -0
  238. package/src/hosted/mcp-http.js +71 -0
  239. package/src/hosted/membership.js +114 -0
  240. package/src/hosted/operator.js +478 -0
  241. package/src/hosted/ops-cli.js +345 -0
  242. package/src/hosted/pages.js +146 -0
  243. package/src/hosted/permissions.js +113 -0
  244. package/src/hosted/presence.js +372 -0
  245. package/src/hosted/projects.js +124 -0
  246. package/src/hosted/public.js +109 -0
  247. package/src/hosted/push.js +622 -0
  248. package/src/hosted/quotas.js +115 -0
  249. package/src/hosted/ratelimit.js +96 -0
  250. package/src/hosted/rebind.js +316 -0
  251. package/src/hosted/retention.js +128 -0
  252. package/src/hosted/server.js +260 -0
  253. package/src/hosted/setup-link.js +64 -0
  254. package/src/hosted/setup.js +323 -0
  255. package/src/hosted/shares.js +444 -0
  256. package/src/hosted/tailscale.js +204 -0
  257. package/src/hosted/tenancy.js +301 -0
  258. package/src/hosted/token-api.js +455 -0
  259. package/src/hosted/token-events.js +60 -0
  260. package/src/hosted/tokens.js +57 -0
  261. package/src/hosted/turnstile.js +49 -0
  262. package/src/hosted/usercontent.js +154 -0
  263. package/src/hosted/zip.js +155 -0
  264. package/src/invocation.js +58 -0
  265. package/src/journey-paths.js +282 -0
  266. package/src/journeys.js +189 -0
  267. package/src/links.js +106 -0
  268. package/src/local-projects.js +182 -0
  269. package/src/lock.js +193 -0
  270. package/src/mcp.js +417 -0
  271. package/src/mentions.js +151 -0
  272. package/src/net.js +15 -0
  273. package/src/notify/config.js +275 -0
  274. package/src/notify/index.js +301 -0
  275. package/src/notify/lease.js +73 -0
  276. package/src/notify/messages.js +212 -0
  277. package/src/notify/run.js +174 -0
  278. package/src/pixeldiff.js +49 -0
  279. package/src/png.js +143 -0
  280. package/src/reactions.js +111 -0
  281. package/src/remote/cli.js +133 -0
  282. package/src/remote/credentials.js +62 -0
  283. package/src/remote/ids.js +12 -0
  284. package/src/remote/login.js +155 -0
  285. package/src/remote/remote-mcp.js +237 -0
  286. package/src/remote/remote-watch.js +291 -0
  287. package/src/remote/remote.js +313 -0
  288. package/src/remote/share-state.js +78 -0
  289. package/src/remote/share.js +369 -0
  290. package/src/remote/terminal.js +13 -0
  291. package/src/serve-state.js +64 -0
  292. package/src/server/app.js +824 -0
  293. package/src/server/devices.js +186 -0
  294. package/src/server/local-console.js +28 -0
  295. package/src/server/local-user.js +59 -0
  296. package/src/server/pairing-routes.js +159 -0
  297. package/src/server/share-routes.js +89 -0
  298. package/src/server/tailscale.js +97 -0
  299. package/src/sitemap.js +431 -0
  300. package/src/store.js +983 -0
  301. package/src/tokens.js +11 -0
  302. package/src/update/routes.js +245 -0
  303. package/src/watch.js +403 -0
  304. package/src/which.js +58 -0
@@ -0,0 +1,21 @@
1
+ import { Devices } from './server/devices.js'
2
+ export function devicesCommand(config, positional, flags, out, env = process.env) {
3
+ const devices = new Devices(config.dir, { env })
4
+ const [action = 'list', id] = positional
5
+ if (action === 'list') {
6
+ const list = devices.list()
7
+ if (flags.json) out(JSON.stringify(list, null, 2))
8
+ else {
9
+ for (const item of list.devices) out(`${item.id} ${item.name} (last seen ${new Date(item.lastSeenAt).toISOString()})`)
10
+ for (const item of list.pending) out(`Pending ${item.code.slice(0, 4)}-${item.code.slice(4)} ${item.name} (${item.ip})`)
11
+ if (!list.devices.length && !list.pending.length) out('No paired devices or pending requests for this project.')
12
+ }
13
+ } else if (['approve', 'deny'].includes(action)) {
14
+ if (!id) throw new Error(`devices ${action} needs the code shown on the device.`)
15
+ const result = devices.decide(id, action === 'approve')
16
+ out(`${result.name}: ${result.state}`)
17
+ } else if (action === 'revoke') {
18
+ if (!id && !flags.all) throw new Error('devices revoke needs a device id or --all.')
19
+ out(`Revoked ${devices.revoke(flags.all ? '--all' : id)} device(s).`)
20
+ } else throw new Error('devices: use list, approve CODE, deny CODE, or revoke ID|--all.')
21
+ }
package/src/diff.js ADDED
@@ -0,0 +1,95 @@
1
+ // Pixel diffs between two environments' captures of the same page and
2
+ // viewport (pixelmatch). Computed when the board asks for a comparison (and
3
+ // after an environment capture), then cached under
4
+ // <board>/diffs/<a>_<b>/: a transparent PNG with the changed pixels in red,
5
+ // to lay over the screenshot, and index.json with the changed share. A cached
6
+ // entry is reused while both source files are unchanged (size and mtime).
7
+
8
+ import fs from 'node:fs'
9
+ import path from 'node:path'
10
+ import { PNG } from 'pngjs'
11
+ import { withFileLock } from './lock.js'
12
+ import { DIFF_THRESHOLD, diffPixels } from './pixeldiff.js'
13
+ import { capturesDir, diffsDir, readJson, writeFileAtomic, writeJsonAtomic } from './store.js'
14
+
15
+ const CACHE_FORMAT = 1
16
+
17
+ // Compares two PNG buffers (see diffPixels). Returns { width, height,
18
+ // changed, total, percent, png } where png is the diff mask (red where
19
+ // changed, transparent elsewhere).
20
+ export function diffPngs(bufferA, bufferB, { threshold = DIFF_THRESHOLD } = {}) {
21
+ const { data, ...result } = diffPixels(PNG.sync.read(bufferA), PNG.sync.read(bufferB), { threshold })
22
+ const mask = new PNG({ width: result.width, height: result.height })
23
+ mask.data.set(data)
24
+ return { ...result, png: PNG.sync.write(mask) }
25
+ }
26
+
27
+ function stamp(file) {
28
+ try {
29
+ const stat = fs.statSync(file)
30
+ return `${stat.size}:${Math.round(stat.mtimeMs)}`
31
+ } catch {
32
+ return null
33
+ }
34
+ }
35
+
36
+ const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve))
37
+
38
+ // Compares environment `a` with `b` for every page and viewport of the board.
39
+ // Returns { a, b, pages: { [pageId]: { [viewport]: { status, percent?, diff? } } }, changedPages }
40
+ // status: 'ok' (diff is the mask file name), 'missing' (one side has no capture).
41
+ export async function compareEnvironments(config, board, a, b, { onlyPages = null } = {}) {
42
+ const dir = diffsDir(config, board.id, a, b)
43
+ const indexFile = path.join(dir, 'index.json')
44
+ const dirA = capturesDir(config, board.id, a)
45
+ const dirB = capturesDir(config, board.id, b)
46
+ return withFileLock(
47
+ indexFile,
48
+ async () => {
49
+ const cached = readJson(indexFile)
50
+ const entries = cached?.format === CACHE_FORMAT && cached.entries && typeof cached.entries === 'object' ? cached.entries : {}
51
+ const pages = {}
52
+ let dirty = false
53
+ for (const page of board.pages) {
54
+ if (onlyPages && !onlyPages.includes(page.id)) continue
55
+ pages[page.id] = {}
56
+ for (const viewport of page.viewports) {
57
+ const name = `${page.id}.${viewport}.png`
58
+ const fileA = path.join(dirA, name)
59
+ const fileB = path.join(dirB, name)
60
+ const key = `${page.id}.${viewport}`
61
+ const sa = stamp(fileA)
62
+ const sb = stamp(fileB)
63
+ if (!sa || !sb) {
64
+ pages[page.id][viewport] = { status: 'missing' }
65
+ continue
66
+ }
67
+ const hit = entries[key]
68
+ if (hit && hit.a === sa && hit.b === sb && fs.existsSync(path.join(dir, name))) {
69
+ pages[page.id][viewport] = { status: 'ok', percent: hit.percent, diff: name }
70
+ continue
71
+ }
72
+ let result
73
+ try {
74
+ result = diffPngs(fs.readFileSync(fileA), fs.readFileSync(fileB))
75
+ } catch {
76
+ pages[page.id][viewport] = { status: 'missing' }
77
+ continue
78
+ }
79
+ writeFileAtomic(path.join(dir, name), result.png)
80
+ entries[key] = { a: sa, b: sb, percent: result.percent, changed: result.changed, total: result.total }
81
+ dirty = true
82
+ pages[page.id][viewport] = { status: 'ok', percent: result.percent, diff: name }
83
+ // Big boards: let the server answer other requests between pages.
84
+ await yieldToLoop()
85
+ }
86
+ }
87
+ if (dirty) writeJsonAtomic(indexFile, { format: CACHE_FORMAT, entries })
88
+ const changedPages = Object.values(pages).filter((viewports) =>
89
+ Object.values(viewports).some((item) => item.status === 'ok' && item.percent > 0),
90
+ ).length
91
+ return { a, b, pages, changedPages }
92
+ },
93
+ { timeoutMs: 120000, staleMs: 300000 },
94
+ )
95
+ }
@@ -0,0 +1,284 @@
1
+ // Page discovery for `shapewall init --base-url`: finds a site's real pages
2
+ // by following the links on its home page (two levels deep at most), so a
3
+ // new config starts with pages that exist. Plain GETs only: no cookies, no
4
+ // JavaScript, no other origin (also after redirects), HTML responses only,
5
+ // with a page cap, a per-request timeout, a total time budget and a body cap.
6
+ // The base URL's own origin may be localhost or a private address: this is
7
+ // meant for the user's own dev server.
8
+
9
+ import { readBounded } from './sitemap.js'
10
+
11
+ export const DISCOVER_DEFAULTS = Object.freeze({
12
+ maxPages: 25,
13
+ maxDepth: 2,
14
+ totalMs: 10000,
15
+ requestMs: 4000,
16
+ bytesMax: 2 * 1024 * 1024,
17
+ concurrency: 4,
18
+ })
19
+
20
+ const REDIRECTS_MAX = 5
21
+ const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])
22
+ // Links to files, not pages.
23
+ const FILE_EXT =
24
+ /\.(pdf|zip|gz|tgz|tar|rar|7z|dmg|exe|msi|apk|png|jpe?g|gif|webp|avif|svg|ico|bmp|tiff?|css|js|mjs|map|xml|json|txt|csv|rss|atom|mp3|mp4|m4a|webm|mov|wav|ogg|woff2?|ttf|otf|eot|docx?|xlsx?|pptx?)$/i
25
+
26
+ const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', ndash: '-', mdash: '-', middot: '·', hellip: '...' }
27
+
28
+ function decodeHtml(value) {
29
+ return String(value).replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, name) => {
30
+ if (name[0] === '#') {
31
+ const code = name[1].toLowerCase() === 'x' ? parseInt(name.slice(2), 16) : parseInt(name.slice(1), 10)
32
+ return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : ''
33
+ }
34
+ return ENTITIES[name.toLowerCase()] ?? match
35
+ })
36
+ }
37
+
38
+ const clean = (value) => decodeHtml(value).replace(/\s+/g, ' ').trim()
39
+
40
+ // Comments, scripts, styles and templates hold text that looks like markup.
41
+ const stripNoise = (html) =>
42
+ String(html)
43
+ .replace(/<!--[\s\S]*?-->/g, '')
44
+ .replace(/<(script|style|template|noscript)\b[\s\S]*?<\/\1\s*>/gi, '')
45
+
46
+ const attr = (tag, name) => {
47
+ const match = new RegExp(`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'>]+))`, 'i').exec(tag)
48
+ return match ? decodeHtml(match[1] ?? match[2] ?? match[3] ?? '').trim() : null
49
+ }
50
+
51
+ // The page's <title>, whitespace collapsed, or null.
52
+ export function htmlTitle(html) {
53
+ const match = /<title\b[^>]*>([\s\S]*?)<\/title\s*>/i.exec(String(html).replace(/<!--[\s\S]*?-->/g, ''))
54
+ const title = match ? clean(match[1]) : ''
55
+ return title || null
56
+ }
57
+
58
+ // The href of every <a> in the page, resolved against the page URL (or its
59
+ // <base href>), as absolute URLs. Unparseable hrefs are dropped.
60
+ export function htmlLinks(html, pageUrl) {
61
+ const source = stripNoise(html)
62
+ let base = pageUrl
63
+ const baseTag = /<base\b[^>]*>/i.exec(source)
64
+ const baseHref = baseTag && attr(baseTag[0], 'href')
65
+ if (baseHref) {
66
+ try {
67
+ base = new URL(baseHref, pageUrl).toString()
68
+ } catch {
69
+ // keep the page URL
70
+ }
71
+ }
72
+ const links = []
73
+ for (const match of source.matchAll(/<a\b[^>]*>/gi)) {
74
+ const href = attr(match[0], 'href')
75
+ if (!href) continue
76
+ try {
77
+ links.push(new URL(href, base))
78
+ } catch {
79
+ // not a URL
80
+ }
81
+ }
82
+ return links
83
+ }
84
+
85
+ // The page a link points at on this site, as a URL string without query
86
+ // and fragment, or null when it leaves the site (another origin, outside
87
+ // the base path), is not http(s), or names a file.
88
+ export function pageLink(url, { origin, basePath = '' }) {
89
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') return null
90
+ if (url.origin !== origin) return null
91
+ if (basePath && url.pathname !== basePath && !url.pathname.startsWith(`${basePath}/`)) return null
92
+ if (FILE_EXT.test(url.pathname)) return null
93
+ const next = new URL(url)
94
+ next.search = ''
95
+ next.hash = ''
96
+ return next.toString()
97
+ }
98
+
99
+ // "/about" and "/about/" are one page.
100
+ const pageKey = (value) => {
101
+ const url = new URL(value)
102
+ return `${url.origin}${url.pathname.replace(/\/+$/, '') || '/'}`
103
+ }
104
+
105
+ // A failed request in a few words ("connection refused (is the site running?)").
106
+ export function describeFetchError(error) {
107
+ const code = error?.cause?.code || error?.code
108
+ if (error?.name === 'TimeoutError' || error?.name === 'AbortError' || code === 'UND_ERR_HEADERS_TIMEOUT') return 'timed out'
109
+ if (code === 'ECONNREFUSED') return 'connection refused (is the site running?)'
110
+ if (code === 'ENOTFOUND' || code === 'EAI_AGAIN') return 'host not found'
111
+ if (code === 'ECONNRESET') return 'connection reset'
112
+ return error?.cause?.message || error?.message || String(error)
113
+ }
114
+
115
+ // One page: GET, redirects followed by hand within the origin, 2xx HTML only.
116
+ // Returns { url, html } or { error }.
117
+ async function fetchPage(start, { fetch, origin, signal, requestMs, bytesMax }) {
118
+ let url = start
119
+ try {
120
+ for (let hops = 0; ; hops += 1) {
121
+ const response = await fetch(url, {
122
+ method: 'GET',
123
+ headers: { accept: 'text/html, application/xhtml+xml;q=0.9, */*;q=0.1', 'user-agent': 'Shapewall init' },
124
+ redirect: 'manual',
125
+ credentials: 'omit',
126
+ signal: AbortSignal.any([signal, AbortSignal.timeout(requestMs)]),
127
+ })
128
+ if (REDIRECT_STATUSES.has(response.status)) {
129
+ await response.body?.cancel?.().catch(() => {})
130
+ const location = response.headers?.get?.('location')
131
+ if (!location) return { error: `HTTP ${response.status} without a Location` }
132
+ let next
133
+ try {
134
+ next = new URL(location, url)
135
+ } catch {
136
+ return { error: 'redirected to an invalid URL' }
137
+ }
138
+ if (next.origin !== origin) return { error: `redirected to another site (${next.origin})` }
139
+ if (hops >= REDIRECTS_MAX) return { error: `more than ${REDIRECTS_MAX} redirects` }
140
+ url = next.toString()
141
+ continue
142
+ }
143
+ if (response.url && new URL(response.url).origin !== origin) return { error: 'redirected to another site' }
144
+ if (!response.ok) {
145
+ await response.body?.cancel?.().catch(() => {})
146
+ return { error: `HTTP ${response.status}` }
147
+ }
148
+ const type = String(response.headers?.get?.('content-type') || '').toLowerCase()
149
+ if (type && !/text\/html|application\/xhtml\+xml/.test(type)) {
150
+ await response.body?.cancel?.().catch(() => {})
151
+ return { error: `not a web page (${type.split(';')[0]})` }
152
+ }
153
+ const html = (await readBounded(response, url, bytesMax)).toString('utf8')
154
+ if (!type && !/<(!doctype html|html|head|body|title)\b/i.test(html)) return { error: 'not a web page' }
155
+ return { url, html }
156
+ }
157
+ } catch (error) {
158
+ return { error: describeFetchError(error) }
159
+ }
160
+ }
161
+
162
+ // Runs `task` over `items` with at most `limit` at a time, results in order.
163
+ async function pool(items, limit, task) {
164
+ const results = new Array(items.length)
165
+ let next = 0
166
+ const worker = async () => {
167
+ while (next < items.length) {
168
+ const index = next
169
+ next += 1
170
+ results[index] = await task(items[index], index)
171
+ }
172
+ }
173
+ await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker))
174
+ return results
175
+ }
176
+
177
+ // Crawls from the base URL. Returns { pages: [{ url, title }], homeTitle,
178
+ // home: { ok, error }, failed: [{ url, reason }], stopped } where pages are
179
+ // the reachable HTML pages in discovery order (home first), `failed` lists
180
+ // links that did not give a page, and `stopped` says why discovery ended
181
+ // early ('pages' at the page cap, 'time' at the time budget), or null.
182
+ export async function discoverPages(baseUrl, { fetch = globalThis.fetch, ...options } = {}) {
183
+ const settings = { ...DISCOVER_DEFAULTS, ...options }
184
+ const base = new URL(baseUrl)
185
+ base.search = ''
186
+ base.hash = ''
187
+ const origin = base.origin
188
+ const basePath = base.pathname.replace(/\/+$/, '')
189
+ const signal = AbortSignal.timeout(settings.totalMs)
190
+ const seen = new Set([pageKey(base.toString())])
191
+ const pages = []
192
+ const failed = []
193
+ let fetches = 0
194
+ let stopped = null
195
+ let homeTitle = null
196
+ let home = { ok: false, error: null }
197
+ // Each level holds links in the order they were found, so results do not
198
+ // depend on which response came back first.
199
+ let level = [base.toString()]
200
+ for (let depth = 0; depth <= settings.maxDepth && level.length; depth += 1) {
201
+ // Pages found so far, counting this level's as they come in.
202
+ let found = pages.length
203
+ const results = await pool(level, settings.concurrency, async (url) => {
204
+ if (found >= settings.maxPages || fetches >= settings.maxPages * 2) {
205
+ stopped ||= 'pages'
206
+ return null
207
+ }
208
+ if (signal.aborted) {
209
+ stopped ||= 'time'
210
+ return null
211
+ }
212
+ fetches += 1
213
+ const result = await fetchPage(url, { fetch, origin, signal, ...settings })
214
+ if (signal.aborted && result.error) stopped ||= 'time'
215
+ if (!result.error) found += 1
216
+ return result
217
+ })
218
+ const nextLevel = []
219
+ results.forEach((result, index) => {
220
+ if (!result) return
221
+ const url = level[index]
222
+ if (result.error) {
223
+ if (depth === 0) home = { ok: false, error: result.error }
224
+ else failed.push({ url, reason: result.error })
225
+ return
226
+ }
227
+ if (pages.length >= settings.maxPages) {
228
+ stopped ||= 'pages'
229
+ return
230
+ }
231
+ const title = htmlTitle(result.html)
232
+ if (depth === 0) {
233
+ home = { ok: true, error: null }
234
+ homeTitle = title
235
+ pages.push({ url, title })
236
+ } else {
237
+ // A link that redirected lands on the page it redirected to.
238
+ const finalKey = pageKey(result.url)
239
+ if (result.url !== url && seen.has(finalKey) && finalKey !== pageKey(url)) return
240
+ seen.add(finalKey)
241
+ pages.push({ url: pageLink(new URL(result.url), { origin, basePath }) || url, title })
242
+ }
243
+ if (depth >= settings.maxDepth) return
244
+ for (const link of htmlLinks(result.html, result.url)) {
245
+ const page = pageLink(link, { origin, basePath })
246
+ if (!page) continue
247
+ const key = pageKey(page)
248
+ if (seen.has(key)) continue
249
+ seen.add(key)
250
+ nextLevel.push(page)
251
+ }
252
+ })
253
+ if (stopped === 'time') break
254
+ level = nextLevel
255
+ }
256
+ return { pages, homeTitle, home, failed, stopped }
257
+ }
258
+
259
+ // A title cut at a word boundary near `max` characters.
260
+ export function cutTitle(value, max = 60) {
261
+ const text = String(value || '').trim()
262
+ if (text.length <= max) return text
263
+ const cut = text.slice(0, max + 1)
264
+ const space = cut.lastIndexOf(' ')
265
+ return `${(space > max / 2 ? cut.slice(0, space) : text.slice(0, max)).replace(/[\s|:,;.-]+$/, '')}...`
266
+ }
267
+
268
+ // Between a page's name and the site's in a <title>: bar, hyphen, en and em dash (as code points), dot, colon.
269
+ const TITLE_SEPARATORS = [' | ', ' - ', ` ${String.fromCodePoint(0x2013)} `, ` ${String.fromCodePoint(0x2014)} `, ' · ', ' :: ', ': ']
270
+
271
+ // A page's own title without the site name around it ("Pricing | Acme" is
272
+ // "Pricing" on the Acme board), or null when nothing of its own is left.
273
+ export function pageTitle(title, siteTitle) {
274
+ let text = String(title || '').trim()
275
+ if (!text) return null
276
+ if (siteTitle && text !== siteTitle) {
277
+ for (const sep of TITLE_SEPARATORS) {
278
+ if (text.endsWith(`${sep}${siteTitle}`)) text = text.slice(0, -(sep.length + siteTitle.length)).trim()
279
+ else if (text.startsWith(`${siteTitle}${sep}`)) text = text.slice(siteTitle.length + sep.length).trim()
280
+ }
281
+ }
282
+ if (!text || text === siteTitle) return null
283
+ return cutTitle(text, 120)
284
+ }