shapewall 0.0.0-stage → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +139 -0
- package/Dockerfile.agent +45 -0
- package/LICENSE +202 -0
- package/NOTICE +12 -0
- package/README.md +651 -2
- package/THIRD_PARTY.md +165 -0
- package/bin/shapewall.js +29 -0
- package/dist/agent/package-lock.json +8384 -0
- package/dist/client/assets/BoardChat-D5SFd8HE.js +28 -0
- package/dist/client/assets/BuzzDialog-HWXBy52W.js +1 -0
- package/dist/client/assets/ChecksPanel-DTdImWoD.js +1 -0
- package/dist/client/assets/History-CEv6b-av.js +1 -0
- package/dist/client/assets/LocalSettings-C4vriEVQ.js +1 -0
- package/dist/client/assets/Management-DCfhcEUY.js +1 -0
- package/dist/client/assets/QuickConnect-DQGgUsl1.js +1 -0
- package/dist/client/assets/ShareDialog-C_plIto0.js +2 -0
- package/dist/client/assets/app-update-dialog-2hPz_N9G.js +1 -0
- package/dist/client/assets/circle-x-BZMPi8Zf.js +1 -0
- package/dist/client/assets/index-ByQRKVit.css +2 -0
- package/dist/client/assets/index-DweQ8Qtu.js +29 -0
- package/dist/client/assets/refresh-cw-CRow0Nbq.js +1 -0
- package/dist/client/assets/runtime-Bft6x_6E.js +4 -0
- package/dist/client/assets/square-A8EnIYLR.js +1 -0
- package/dist/client/assets/tts-dist-DYlBQmwG.js +18 -0
- package/dist/client/assets/tts-en-g2p-BPUtBFGG.js +1 -0
- package/dist/client/assets/tts-ort-wasm-simd-threaded-qh98TCA_.wasm +0 -0
- package/dist/client/assets/tts-ort.wasm.bundle.min-rEkcs_ow.js +12 -0
- package/dist/client/assets/tts-rolldown-runtime-D8NudN84.js +1 -0
- package/dist/client/assets/tts-spacy-en-tokenizer-6SNMl--A.json +1 -0
- package/dist/client/assets/tts-tts.worker-Bzm6aWbv.js +1 -0
- package/dist/client/assets/tts-us_gold-DcjeNMH1.json +92589 -0
- package/dist/client/assets/tts-us_silver-C964mEB5.json +93363 -0
- package/dist/client/assets/ui-CLbws0rx.js +41 -0
- package/dist/client/build-info.json +1 -0
- package/dist/client/favicon.svg +7 -0
- package/dist/client/icons/apple-touch-icon.png +0 -0
- package/dist/client/icons/icon-192.png +0 -0
- package/dist/client/icons/icon-512.png +0 -0
- package/dist/client/icons/maskable-512.png +0 -0
- package/dist/client/index.html +25 -0
- package/dist/client/manifest.webmanifest +15 -0
- package/dist/client/sw.js +53 -0
- package/dist/client/theme-boot.js +27 -0
- package/dist/console/assets/LocalAccountsApp-BQCcmj3x.js +1 -0
- package/dist/console/assets/LocalConsole-Bb2FUDs7.js +1 -0
- package/dist/console/assets/app-update-dialog-DIeYepLZ.js +1 -0
- package/dist/console/assets/console-C7uJLLOn.js +51 -0
- package/dist/console/assets/console-R8IBY3Am.css +2 -0
- package/dist/console/build-info.json +1 -0
- package/dist/console/console.html +21 -0
- package/dist/console/favicon.svg +7 -0
- package/dist/console/icons/apple-touch-icon.png +0 -0
- package/dist/console/icons/icon-192.png +0 -0
- package/dist/console/icons/icon-512.png +0 -0
- package/dist/console/icons/maskable-512.png +0 -0
- package/dist/console/manifest.webmanifest +15 -0
- package/dist/console/theme-boot.js +27 -0
- package/docs/agent-hardening.md +342 -0
- package/docs/agent-recipes.md +187 -0
- package/docs/agent-workflow.md +320 -0
- package/docs/agents.md +894 -0
- package/docs/architecture.md +242 -0
- package/docs/buzz.md +140 -0
- package/docs/capture-warnings.md +57 -0
- package/docs/config.md +489 -0
- package/docs/deployment.md +289 -0
- package/docs/design-system.md +162 -0
- package/docs/github.md +107 -0
- package/docs/hosted.md +764 -0
- package/docs/images/board-overview.png +0 -0
- package/docs/images/journeys.png +0 -0
- package/docs/images/live-comment.png +0 -0
- package/docs/images/phone-comment.png +0 -0
- package/docs/images/phone-overview.png +0 -0
- package/docs/images/variant.png +0 -0
- package/docs/interaction-audits.md +40 -0
- package/docs/journeys.md +137 -0
- package/docs/local-accounts.md +80 -0
- package/docs/notifications.md +237 -0
- package/docs/page-checks.md +69 -0
- package/docs/performance.md +174 -0
- package/docs/starter-agent.md +325 -0
- package/docs/updates.md +87 -0
- package/examples/audits/interaction.audit.json +17 -0
- package/package.json +132 -4
- package/src/accounts/avatar.js +121 -0
- package/src/accounts/local.js +464 -0
- package/src/agent/acp/command.js +204 -0
- package/src/agent/acp/detect.js +75 -0
- package/src/agent/acp/sandbox.js +160 -0
- package/src/agent/agent-health.js +75 -0
- package/src/agent/backends/acp.js +1297 -0
- package/src/agent/backends/index.js +61 -0
- package/src/agent/backends/openai.js +308 -0
- package/src/agent/composer.js +221 -0
- package/src/agent/config.js +338 -0
- package/src/agent/context.js +191 -0
- package/src/agent/control.js +122 -0
- package/src/agent/errors.js +17 -0
- package/src/agent/feedback-loop.js +137 -0
- package/src/agent/manage.js +94 -0
- package/src/agent/next-steps.js +51 -0
- package/src/agent/owner-mcp.js +38 -0
- package/src/agent/plain-text.js +52 -0
- package/src/agent/ratings-core.js +213 -0
- package/src/agent/ratings.js +100 -0
- package/src/agent/reply-text.js +67 -0
- package/src/agent/routes.js +1065 -0
- package/src/agent/run.js +326 -0
- package/src/agent/settings-routes.js +704 -0
- package/src/agent/showable.js +23 -0
- package/src/agent/starter/approvals.js +181 -0
- package/src/agent/starter/build-image.js +90 -0
- package/src/agent/starter/cli.js +163 -0
- package/src/agent/starter/disk.js +53 -0
- package/src/agent/starter/doctor.js +180 -0
- package/src/agent/starter/egress-proxy.js +351 -0
- package/src/agent/starter/gate.js +90 -0
- package/src/agent/starter/loop.js +114 -0
- package/src/agent/starter/model-relay.js +289 -0
- package/src/agent/starter/pause.js +14 -0
- package/src/agent/starter/probe.js +272 -0
- package/src/agent/starter/prompt.js +41 -0
- package/src/agent/starter/push-guard.js +146 -0
- package/src/agent/starter/rpc.js +164 -0
- package/src/agent/starter/runtime.js +287 -0
- package/src/agent/starter/sandbox-main.js +121 -0
- package/src/agent/starter/settings.js +290 -0
- package/src/agent/starter/source.js +69 -0
- package/src/agent/starter/supervisor.js +755 -0
- package/src/agent/starter/sweep.js +46 -0
- package/src/agent/starter/tools.js +524 -0
- package/src/agent/starter/watch.js +84 -0
- package/src/agent/starter/worktree.js +409 -0
- package/src/agent/text-diff.js +210 -0
- package/src/agent/thread-history.js +223 -0
- package/src/agent/threads.js +399 -0
- package/src/agent/tools.js +200 -0
- package/src/agent/turn-changes.js +424 -0
- package/src/agent/undo-state.js +26 -0
- package/src/agent/user-settings.js +308 -0
- package/src/auth/record.js +375 -0
- package/src/broken-links.js +164 -0
- package/src/buzz/client.js +75 -0
- package/src/buzz/config.js +151 -0
- package/src/buzz/environment.js +13 -0
- package/src/buzz/gateway.js +192 -0
- package/src/buzz/routes.js +61 -0
- package/src/buzz/signer-child.js +205 -0
- package/src/buzz/store.js +89 -0
- package/src/buzz/transport.js +60 -0
- package/src/buzz/verify.js +140 -0
- package/src/capture/actions.js +159 -0
- package/src/capture/capture.js +776 -0
- package/src/capture/dom-names.js +189 -0
- package/src/capture/element-map.js +509 -0
- package/src/capture/live.js +827 -0
- package/src/capture/media-scheme.js +317 -0
- package/src/capture/network-summary.js +86 -0
- package/src/capture/pacing.js +34 -0
- package/src/capture/requests.js +167 -0
- package/src/capture/sanitize.js +752 -0
- package/src/checks/accessibility.js +134 -0
- package/src/checks/budgets.js +58 -0
- package/src/checks/cli.js +55 -0
- package/src/checks/comparison.js +113 -0
- package/src/checks/environments.js +32 -0
- package/src/checks/flow-config.js +82 -0
- package/src/checks/flow-origin.js +53 -0
- package/src/checks/flows.js +398 -0
- package/src/checks/lighthouse-auth.js +145 -0
- package/src/checks/lighthouse-findings.js +130 -0
- package/src/checks/lighthouse.js +395 -0
- package/src/checks/robots.js +85 -0
- package/src/checks/routes.js +52 -0
- package/src/checks/run-control.js +259 -0
- package/src/checks/runner.js +724 -0
- package/src/checks/schema.js +133 -0
- package/src/checks/seo-analysis.js +443 -0
- package/src/checks/seo-probe.js +190 -0
- package/src/checks/seo-snapshot.js +99 -0
- package/src/checks/store.js +214 -0
- package/src/checks/transport.js +47 -0
- package/src/cli/args.js +319 -0
- package/src/cli/capture.js +79 -0
- package/src/cli/commands.js +92 -0
- package/src/cli/comments.js +84 -0
- package/src/cli/help.js +243 -0
- package/src/cli/init.js +214 -0
- package/src/cli/misc.js +64 -0
- package/src/cli/notify.js +87 -0
- package/src/cli/open.js +22 -0
- package/src/cli/serve.js +274 -0
- package/src/cli/variant.js +82 -0
- package/src/cli/watch.js +70 -0
- package/src/cli.js +130 -0
- package/src/comment-fields.js +76 -0
- package/src/comments-cli.js +208 -0
- package/src/config-reload.js +100 -0
- package/src/config.js +1061 -0
- package/src/devices-cli.js +21 -0
- package/src/diff.js +95 -0
- package/src/discover.js +284 -0
- package/src/doctor.js +295 -0
- package/src/event-log.js +268 -0
- package/src/events-cli.js +102 -0
- package/src/events.js +113 -0
- package/src/export.js +119 -0
- package/src/feedback.js +246 -0
- package/src/github.js +551 -0
- package/src/hosted/LICENSE.md +105 -0
- package/src/hosted/access-cli.js +78 -0
- package/src/hosted/admin.js +682 -0
- package/src/hosted/agent-approvals.js +163 -0
- package/src/hosted/agent-policy.js +189 -0
- package/src/hosted/app.js +510 -0
- package/src/hosted/auth-routes.js +540 -0
- package/src/hosted/auth.js +536 -0
- package/src/hosted/backup.js +483 -0
- package/src/hosted/board-context.js +35 -0
- package/src/hosted/board-routes.js +486 -0
- package/src/hosted/board-view.js +94 -0
- package/src/hosted/clean-worker.js +24 -0
- package/src/hosted/cleaner.js +89 -0
- package/src/hosted/comments.js +355 -0
- package/src/hosted/console-routes.js +401 -0
- package/src/hosted/cookies.js +18 -0
- package/src/hosted/crypto.js +80 -0
- package/src/hosted/db.js +496 -0
- package/src/hosted/device-routes.js +175 -0
- package/src/hosted/device.js +233 -0
- package/src/hosted/events.js +63 -0
- package/src/hosted/guards.js +172 -0
- package/src/hosted/guest-view.js +65 -0
- package/src/hosted/http.js +39 -0
- package/src/hosted/import.js +300 -0
- package/src/hosted/mail.js +73 -0
- package/src/hosted/mcp-http.js +71 -0
- package/src/hosted/membership.js +114 -0
- package/src/hosted/operator.js +478 -0
- package/src/hosted/ops-cli.js +345 -0
- package/src/hosted/pages.js +146 -0
- package/src/hosted/permissions.js +113 -0
- package/src/hosted/presence.js +372 -0
- package/src/hosted/projects.js +124 -0
- package/src/hosted/public.js +109 -0
- package/src/hosted/push.js +622 -0
- package/src/hosted/quotas.js +115 -0
- package/src/hosted/ratelimit.js +96 -0
- package/src/hosted/rebind.js +316 -0
- package/src/hosted/retention.js +128 -0
- package/src/hosted/server.js +260 -0
- package/src/hosted/setup-link.js +64 -0
- package/src/hosted/setup.js +323 -0
- package/src/hosted/shares.js +444 -0
- package/src/hosted/tailscale.js +204 -0
- package/src/hosted/tenancy.js +301 -0
- package/src/hosted/token-api.js +455 -0
- package/src/hosted/token-events.js +60 -0
- package/src/hosted/tokens.js +57 -0
- package/src/hosted/turnstile.js +49 -0
- package/src/hosted/usercontent.js +154 -0
- package/src/hosted/zip.js +155 -0
- package/src/invocation.js +58 -0
- package/src/journey-paths.js +282 -0
- package/src/journeys.js +189 -0
- package/src/links.js +106 -0
- package/src/local-projects.js +182 -0
- package/src/lock.js +193 -0
- package/src/mcp.js +417 -0
- package/src/mentions.js +151 -0
- package/src/net.js +15 -0
- package/src/notify/config.js +275 -0
- package/src/notify/index.js +301 -0
- package/src/notify/lease.js +73 -0
- package/src/notify/messages.js +212 -0
- package/src/notify/run.js +174 -0
- package/src/pixeldiff.js +49 -0
- package/src/png.js +143 -0
- package/src/reactions.js +111 -0
- package/src/remote/cli.js +133 -0
- package/src/remote/credentials.js +62 -0
- package/src/remote/ids.js +12 -0
- package/src/remote/login.js +155 -0
- package/src/remote/remote-mcp.js +237 -0
- package/src/remote/remote-watch.js +291 -0
- package/src/remote/remote.js +313 -0
- package/src/remote/share-state.js +78 -0
- package/src/remote/share.js +369 -0
- package/src/remote/terminal.js +13 -0
- package/src/serve-state.js +64 -0
- package/src/server/app.js +824 -0
- package/src/server/devices.js +186 -0
- package/src/server/local-console.js +28 -0
- package/src/server/local-user.js +59 -0
- package/src/server/pairing-routes.js +159 -0
- package/src/server/share-routes.js +89 -0
- package/src/server/tailscale.js +97 -0
- package/src/sitemap.js +431 -0
- package/src/store.js +983 -0
- package/src/tokens.js +11 -0
- package/src/update/routes.js +245 -0
- package/src/watch.js +403 -0
- package/src/which.js +58 -0
|
@@ -0,0 +1,827 @@
|
|
|
1
|
+
// Live HTML snapshots. In the same browser page as the screenshot, the
|
|
2
|
+
// rendered DOM (so client-rendered content is included) is cloned, its
|
|
3
|
+
// stylesheets are collected in cascade order (responsive @media rules kept)
|
|
4
|
+
// and form values are frozen. Images are re-encoded inside the page from the
|
|
5
|
+
// decoded pixels (canvas, WebP at quality 0.8, at most twice their displayed
|
|
6
|
+
// width), so they are complete and small; only when that is impossible
|
|
7
|
+
// (cross-origin pixels) the browser's downloaded body is used, after checking
|
|
8
|
+
// it is complete. Fonts come from the downloaded bodies with the same check.
|
|
9
|
+
// Identical images are stored once. The result goes through the strict
|
|
10
|
+
// sanitizer (sanitize.js) and is written as <name>.<viewport>.html.
|
|
11
|
+
|
|
12
|
+
import crypto from 'node:crypto'
|
|
13
|
+
import fs from 'node:fs'
|
|
14
|
+
import { BlockList, isIP } from 'node:net'
|
|
15
|
+
import { parse, serialize } from 'parse5'
|
|
16
|
+
import { placeholderSvg, sanitizeCss, sanitizeDocument } from './sanitize.js'
|
|
17
|
+
import { pinnedColorScheme, resolveMediaInCss } from './media-scheme.js'
|
|
18
|
+
import { writeFileAtomic } from '../store.js'
|
|
19
|
+
import { ELEMENT_MAP_EXPRESSION, ELEMENT_MAP_LIMITS, cleanElementMap } from './element-map.js'
|
|
20
|
+
|
|
21
|
+
const IMAGE_BUDGET = 3 * 1024 * 1024 // data: characters for all distinct images on a page
|
|
22
|
+
const IMAGE_MAX_EACH = 640 * 1024 // one image after re-encoding
|
|
23
|
+
const BODY_MAX_EACH = 1536 * 1024 // a downloaded body we would inline as is
|
|
24
|
+
const FONT_BUDGET = 1536 * 1024 // all fonts; above it pages fall back to system fonts
|
|
25
|
+
const FONT_MAX_EACH = 400 * 1024
|
|
26
|
+
const HTML_MAX = 8 * 1024 * 1024 // the page snapshot as returned from the browser
|
|
27
|
+
export const FILE_MAX = 5 * 1024 * 1024 // the final written document
|
|
28
|
+
const CSS_MAX = 6 * 1024 * 1024 // all stylesheet text of one page
|
|
29
|
+
const CSS_SHEET_MAX = 2 * 1024 * 1024 // one cross-origin stylesheet fetched by body
|
|
30
|
+
const CSS_IMAGES_MAX = 60 // background images re-encoded per page
|
|
31
|
+
const FRAME_RULES = 'html,body{overflow:hidden!important;scroll-behavior:auto!important}'
|
|
32
|
+
const TRANSPARENT_GIF = 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7'
|
|
33
|
+
const BASE64_IMAGE_RE = /^data:image\/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+$/
|
|
34
|
+
|
|
35
|
+
// Runs inside the captured page (Runtime.evaluate, awaited). Must be
|
|
36
|
+
// self-contained: it is sent as source text. It does not change the live page.
|
|
37
|
+
export async function snapshotPage(limits) {
|
|
38
|
+
const doc = document
|
|
39
|
+
const base = doc.baseURI
|
|
40
|
+
const absolute = (value) => {
|
|
41
|
+
try {
|
|
42
|
+
return new URL(value, base).href
|
|
43
|
+
} catch (error) {
|
|
44
|
+
return ''
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Rules whose selectors match nothing on this page are dropped (app CSS
|
|
49
|
+
// bundles are mostly other pages). Pseudo-classes and pseudo-elements are
|
|
50
|
+
// ignored for the test, so :hover and ::before rules of present elements stay.
|
|
51
|
+
// An escaped colon is part of a class name (Tailwind's .lg\:translate-x-0),
|
|
52
|
+
// not a pseudo-class, so it is left alone.
|
|
53
|
+
const matches = (selectorText) =>
|
|
54
|
+
selectorText.split(',').some((part) => {
|
|
55
|
+
const plain = part.replace(/(?<!\\)::?[a-zA-Z-]+(\([^)]*\))?/g, '').trim() || '*'
|
|
56
|
+
try {
|
|
57
|
+
return Boolean(doc.querySelector(plain))
|
|
58
|
+
} catch (error) {
|
|
59
|
+
return true
|
|
60
|
+
}
|
|
61
|
+
})
|
|
62
|
+
const ruleText = (rule) => {
|
|
63
|
+
if (rule.type === 1) return matches(rule.selectorText) ? rule.cssText : ''
|
|
64
|
+
if (
|
|
65
|
+
rule.cssRules &&
|
|
66
|
+
(rule.type === 4 ||
|
|
67
|
+
rule.type === 12 ||
|
|
68
|
+
(typeof CSSLayerBlockRule !== 'undefined' && rule instanceof CSSLayerBlockRule) ||
|
|
69
|
+
(typeof CSSContainerRule !== 'undefined' && rule instanceof CSSContainerRule))
|
|
70
|
+
) {
|
|
71
|
+
const inner = Array.from(rule.cssRules).map(ruleText).filter(Boolean).join('\n')
|
|
72
|
+
if (!inner) return ''
|
|
73
|
+
const head = rule.cssText.slice(0, rule.cssText.indexOf('{')).trim()
|
|
74
|
+
return `${head} {\n${inner}\n}`
|
|
75
|
+
}
|
|
76
|
+
return rule.cssText
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// Stylesheets in cascade order; @import sheets come before the rest of
|
|
80
|
+
// their parent, as they apply. Cross-origin sheets are fetched by Node.
|
|
81
|
+
const chunks = []
|
|
82
|
+
const pushSheet = (sheet, depth, inheritedMedia) => {
|
|
83
|
+
if (!sheet || depth > 6 || sheet.disabled) return
|
|
84
|
+
const mediaText = [inheritedMedia, sheet.media && sheet.media.mediaText].filter((text) => text && text !== 'all').join(' and ')
|
|
85
|
+
const sheetBase = sheet.href || base
|
|
86
|
+
let rules = null
|
|
87
|
+
try {
|
|
88
|
+
rules = sheet.cssRules
|
|
89
|
+
} catch (error) {
|
|
90
|
+
rules = null
|
|
91
|
+
}
|
|
92
|
+
if (!rules) {
|
|
93
|
+
chunks.push({ href: sheet.href, base: sheetBase, text: null, media: mediaText })
|
|
94
|
+
return
|
|
95
|
+
}
|
|
96
|
+
let text = ''
|
|
97
|
+
for (const rule of rules) {
|
|
98
|
+
if (rule.type === 3 && rule.styleSheet) {
|
|
99
|
+
pushSheet(
|
|
100
|
+
rule.styleSheet,
|
|
101
|
+
depth + 1,
|
|
102
|
+
[mediaText, rule.media && rule.media.mediaText].filter((part) => part && part !== 'all').join(' and '),
|
|
103
|
+
)
|
|
104
|
+
continue
|
|
105
|
+
}
|
|
106
|
+
const kept = ruleText(rule)
|
|
107
|
+
if (kept) text += `${kept}\n`
|
|
108
|
+
}
|
|
109
|
+
chunks.push({ href: sheet.href, base: sheetBase, text, media: mediaText })
|
|
110
|
+
}
|
|
111
|
+
for (const sheet of Array.from(doc.styleSheets)) pushSheet(sheet, 0, '')
|
|
112
|
+
for (const sheet of Array.from(doc.adoptedStyleSheets || [])) pushSheet(sheet, 0, '')
|
|
113
|
+
const cssLength = chunks.reduce((sum, chunk) => sum + (chunk.text ? chunk.text.length : 0), 0)
|
|
114
|
+
if (cssLength > limits.css) return { tooLarge: `the page CSS is ${Math.round(cssLength / 1048576)} MB, over the cap` }
|
|
115
|
+
|
|
116
|
+
// Re-encode decoded pixels: complete by construction, at most `width`
|
|
117
|
+
// pixels wide. Cross-origin pixels taint the canvas and return null.
|
|
118
|
+
const encode = (image, width) => {
|
|
119
|
+
const naturalWidth = image.naturalWidth || image.width
|
|
120
|
+
const naturalHeight = image.naturalHeight || image.height
|
|
121
|
+
if (!naturalWidth || !naturalHeight) return null
|
|
122
|
+
let targetWidth = Math.max(1, Math.min(naturalWidth, Math.ceil(width)))
|
|
123
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
124
|
+
const targetHeight = Math.max(1, Math.round((targetWidth * naturalHeight) / naturalWidth))
|
|
125
|
+
const canvas = doc.createElement('canvas')
|
|
126
|
+
canvas.width = targetWidth
|
|
127
|
+
canvas.height = targetHeight
|
|
128
|
+
try {
|
|
129
|
+
canvas.getContext('2d').drawImage(image, 0, 0, targetWidth, targetHeight)
|
|
130
|
+
let url = canvas.toDataURL('image/webp', limits.quality)
|
|
131
|
+
if (!url.startsWith('data:image/webp')) url = canvas.toDataURL('image/png')
|
|
132
|
+
if (url.length <= limits.imageEach) return url
|
|
133
|
+
} catch (error) {
|
|
134
|
+
return null
|
|
135
|
+
}
|
|
136
|
+
targetWidth = Math.max(1, Math.round(targetWidth / 2))
|
|
137
|
+
}
|
|
138
|
+
return null
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const clone = doc.documentElement.cloneNode(true)
|
|
142
|
+
const pairs = (selector) => {
|
|
143
|
+
const live = Array.from(doc.documentElement.querySelectorAll(selector))
|
|
144
|
+
const copy = Array.from(clone.querySelectorAll(selector))
|
|
145
|
+
return live.length === copy.length ? live.map((element, index) => [element, copy[index]]) : []
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Freeze what the user would see in form controls.
|
|
149
|
+
for (const [live, copy] of pairs('input, textarea, select')) {
|
|
150
|
+
const tag = live.tagName.toLowerCase()
|
|
151
|
+
if (tag === 'textarea') copy.textContent = live.value
|
|
152
|
+
else if (tag === 'select') {
|
|
153
|
+
Array.from(copy.options).forEach((option, index) => {
|
|
154
|
+
if (live.options[index] && live.options[index].selected) option.setAttribute('selected', '')
|
|
155
|
+
else option.removeAttribute('selected')
|
|
156
|
+
})
|
|
157
|
+
} else if (live.type === 'checkbox' || live.type === 'radio') {
|
|
158
|
+
if (live.checked) copy.setAttribute('checked', '')
|
|
159
|
+
else copy.removeAttribute('checked')
|
|
160
|
+
} else if (live.type === 'hidden' || live.type === 'password') {
|
|
161
|
+
// Tokens and secrets: the Node sanitizer drops these too.
|
|
162
|
+
copy.removeAttribute('value')
|
|
163
|
+
} else if (live.type !== 'file') {
|
|
164
|
+
copy.setAttribute('value', live.value)
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// Images: re-encoded at twice the displayed width; the source URL stays
|
|
169
|
+
// only when the pixels are not readable here (Node then checks the body).
|
|
170
|
+
for (const [live, copy] of pairs('img')) {
|
|
171
|
+
const rect = live.getBoundingClientRect()
|
|
172
|
+
;['srcset', 'sizes', 'loading', 'decoding', 'fetchpriority'].forEach((name) => copy.removeAttribute(name))
|
|
173
|
+
copy.setAttribute('data-sw-w', String(Math.round(rect.width)))
|
|
174
|
+
copy.setAttribute('data-sw-h', String(Math.round(rect.height)))
|
|
175
|
+
const source = live.currentSrc || live.src || ''
|
|
176
|
+
if (!source || !rect.width || !rect.height) {
|
|
177
|
+
copy.removeAttribute('src')
|
|
178
|
+
continue
|
|
179
|
+
}
|
|
180
|
+
const encoded = live.complete ? encode(live, rect.width * 2) : null
|
|
181
|
+
copy.setAttribute('src', encoded || absolute(source))
|
|
182
|
+
}
|
|
183
|
+
clone.querySelectorAll('picture source').forEach((node) => node.remove())
|
|
184
|
+
|
|
185
|
+
// Canvas and video become still images (or sized blanks).
|
|
186
|
+
const still = (live, copy, dataUrl) => {
|
|
187
|
+
const rect = live.getBoundingClientRect()
|
|
188
|
+
const img = doc.createElement('img')
|
|
189
|
+
if (copy.getAttribute('class')) img.setAttribute('class', copy.getAttribute('class'))
|
|
190
|
+
img.setAttribute(
|
|
191
|
+
'style',
|
|
192
|
+
`${copy.getAttribute('style') || ''};width:${Math.round(rect.width)}px;height:${Math.round(rect.height)}px;object-fit:cover`,
|
|
193
|
+
)
|
|
194
|
+
img.setAttribute('data-sw-w', String(Math.round(rect.width)))
|
|
195
|
+
img.setAttribute('data-sw-h', String(Math.round(rect.height)))
|
|
196
|
+
if (dataUrl) img.setAttribute('src', dataUrl)
|
|
197
|
+
copy.replaceWith(img)
|
|
198
|
+
}
|
|
199
|
+
for (const [live, copy] of pairs('canvas')) {
|
|
200
|
+
const rect = live.getBoundingClientRect()
|
|
201
|
+
let dataUrl = null
|
|
202
|
+
try {
|
|
203
|
+
dataUrl = encode(live, rect.width * 2)
|
|
204
|
+
} catch (error) {
|
|
205
|
+
dataUrl = null
|
|
206
|
+
}
|
|
207
|
+
still(live, copy, dataUrl)
|
|
208
|
+
}
|
|
209
|
+
for (const [live, copy] of pairs('video')) still(live, copy, live.poster ? absolute(live.poster) : '')
|
|
210
|
+
|
|
211
|
+
// Background images from same-origin stylesheets, re-encoded the same way
|
|
212
|
+
// (at most the viewport width; their displayed size is unknown here).
|
|
213
|
+
const cssImages = {}
|
|
214
|
+
const wanted = []
|
|
215
|
+
for (const chunk of chunks) {
|
|
216
|
+
if (!chunk.text) continue
|
|
217
|
+
const fontBlocks = []
|
|
218
|
+
const fontRe = /@font-face\s*\{[^}]*\}/gi
|
|
219
|
+
let block
|
|
220
|
+
while ((block = fontRe.exec(chunk.text))) fontBlocks.push([block.index, block.index + block[0].length])
|
|
221
|
+
const urlRe = /url\(\s*(?:"([^"]*)"|'([^']*)'|([^)'"]*))\s*\)/gi
|
|
222
|
+
let match
|
|
223
|
+
while ((match = urlRe.exec(chunk.text)) && wanted.length < limits.cssImages) {
|
|
224
|
+
const raw = (match[1] ?? match[2] ?? match[3] ?? '').trim()
|
|
225
|
+
if (!raw || raw.startsWith('data:') || raw.startsWith('#')) continue
|
|
226
|
+
if (fontBlocks.some(([start, end]) => match.index >= start && match.index < end)) continue
|
|
227
|
+
let url = ''
|
|
228
|
+
try {
|
|
229
|
+
url = new URL(raw, chunk.base).href.split('#')[0]
|
|
230
|
+
} catch (error) {
|
|
231
|
+
url = ''
|
|
232
|
+
}
|
|
233
|
+
if (url && !wanted.includes(url)) wanted.push(url)
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
for (const url of wanted) {
|
|
237
|
+
try {
|
|
238
|
+
const image = new Image()
|
|
239
|
+
image.src = url
|
|
240
|
+
await Promise.race([image.decode(), new Promise((resolve, reject) => setTimeout(() => reject(new Error('slow')), 3000))])
|
|
241
|
+
const encoded = encode(image, Math.min(innerWidth, 1600))
|
|
242
|
+
if (encoded) cssImages[url] = encoded
|
|
243
|
+
} catch (error) {
|
|
244
|
+
/* not loadable here: Node may use the body */
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// Nothing that runs, loads or redirects; styles are rebuilt from `chunks`.
|
|
249
|
+
clone
|
|
250
|
+
.querySelectorAll('script, noscript, iframe, frame, object, embed, base, link, template, style, meta[http-equiv], audio')
|
|
251
|
+
.forEach((node) => node.remove())
|
|
252
|
+
for (const element of Array.from(clone.querySelectorAll('*'))) {
|
|
253
|
+
for (const attr of Array.from(element.attributes)) {
|
|
254
|
+
if (/^on/i.test(attr.name)) element.removeAttribute(attr.name)
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
// Refuse oversized pages here, before anything crosses the DevTools pipe.
|
|
258
|
+
const html = `<!DOCTYPE html>${clone.outerHTML}`
|
|
259
|
+
if (html.length > limits.html) return { tooLarge: `the page HTML is ${Math.round(html.length / 1048576)} MB, over the cap` }
|
|
260
|
+
// The preferences the page was rendered with, so the snapshot can be frozen
|
|
261
|
+
// to them (media-scheme.js): the board's viewer may prefer something else.
|
|
262
|
+
const prefers = (query) => Boolean(window.matchMedia && window.matchMedia(query).matches)
|
|
263
|
+
const scheme = {
|
|
264
|
+
colorScheme: prefers('(prefers-color-scheme: dark)') ? 'dark' : 'light',
|
|
265
|
+
reducedMotion: prefers('(prefers-reduced-motion: reduce)') ? 'reduce' : 'no-preference',
|
|
266
|
+
root: getComputedStyle(doc.documentElement).colorScheme || 'normal',
|
|
267
|
+
}
|
|
268
|
+
return { html, chunks, base, cssImages, title: doc.title, scheme }
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
const SNAPSHOT_LIMITS = { html: HTML_MAX, css: CSS_MAX, quality: 0.8, imageEach: IMAGE_MAX_EACH, cssImages: CSS_IMAGES_MAX }
|
|
272
|
+
export const SNAPSHOT_EXPRESSION = `(${snapshotPage.toString()})(${JSON.stringify(SNAPSHOT_LIMITS)})`
|
|
273
|
+
|
|
274
|
+
// What a snapshot is frozen to. Snapshots from before this was recorded
|
|
275
|
+
// (and synthetic ones) count as light, the capture default; reduced motion is
|
|
276
|
+
// resolved only when it was recorded.
|
|
277
|
+
export function snapshotScheme(snapshot) {
|
|
278
|
+
const recorded = snapshot?.scheme || {}
|
|
279
|
+
return {
|
|
280
|
+
colorScheme: recorded.colorScheme === 'dark' ? 'dark' : 'light',
|
|
281
|
+
...(['reduce', 'no-preference'].includes(recorded.reducedMotion) ? { reducedMotion: recorded.reducedMotion } : {}),
|
|
282
|
+
root: typeof recorded.root === 'string' ? recorded.root.slice(0, 60) : 'normal',
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
function wrapMedia(css, media) {
|
|
287
|
+
return media ? `@media ${media} {\n${css}\n}` : css
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
function stripHash(url) {
|
|
291
|
+
const index = url.indexOf('#')
|
|
292
|
+
return index >= 0 ? url.slice(0, index) : url
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
// ---- Completeness checks ------------------------------------------------------
|
|
296
|
+
// A body or data URI is inlined only if it is the whole file: the format's
|
|
297
|
+
// signature, and an end marker or a length field that matches.
|
|
298
|
+
|
|
299
|
+
export function imageIsComplete(buffer, mime) {
|
|
300
|
+
if (!buffer || buffer.length < 8) return false
|
|
301
|
+
const type = String(mime || '').toLowerCase()
|
|
302
|
+
const startsWith = (bytes, offset = 0) => bytes.every((byte, index) => buffer[offset + index] === byte)
|
|
303
|
+
if (type.includes('png'))
|
|
304
|
+
return startsWith([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) && buffer.subarray(-12).includes(Buffer.from('IEND'))
|
|
305
|
+
if (type.includes('jpeg') || type.includes('jpg')) {
|
|
306
|
+
if (!startsWith([0xff, 0xd8, 0xff])) return false
|
|
307
|
+
let end = buffer.length
|
|
308
|
+
while (end > 2 && buffer[end - 1] === 0x00) end -= 1
|
|
309
|
+
return buffer[end - 2] === 0xff && buffer[end - 1] === 0xd9
|
|
310
|
+
}
|
|
311
|
+
if (type.includes('gif')) return buffer.subarray(0, 4).toString('latin1') === 'GIF8' && buffer[buffer.length - 1] === 0x3b
|
|
312
|
+
if (type.includes('webp'))
|
|
313
|
+
return (
|
|
314
|
+
buffer.subarray(0, 4).toString('latin1') === 'RIFF' &&
|
|
315
|
+
buffer.subarray(8, 12).toString('latin1') === 'WEBP' &&
|
|
316
|
+
buffer.readUInt32LE(4) + 8 <= buffer.length
|
|
317
|
+
)
|
|
318
|
+
if (type.includes('svg')) {
|
|
319
|
+
const text = buffer.toString('utf8')
|
|
320
|
+
return /<svg[\s>]/i.test(text) && /<\/svg>\s*$/i.test(text.trim())
|
|
321
|
+
}
|
|
322
|
+
if (type.includes('avif')) return buffer.subarray(4, 8).toString('latin1') === 'ftyp'
|
|
323
|
+
return false
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
export function fontIsComplete(buffer) {
|
|
327
|
+
if (!buffer || buffer.length < 12) return false
|
|
328
|
+
const tag = buffer.subarray(0, 4).toString('latin1')
|
|
329
|
+
if (tag === 'wOF2' || tag === 'wOFF') return buffer.readUInt32BE(8) === buffer.length
|
|
330
|
+
return tag === 'OTTO' || tag === 'true' || buffer.readUInt32BE(0) === 0x00010000
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
function dataUriBuffer(dataUri) {
|
|
334
|
+
const match = /^data:([^;,]+)(;base64)?,(.*)$/s.exec(dataUri)
|
|
335
|
+
if (!match) return null
|
|
336
|
+
try {
|
|
337
|
+
return { mime: match[1], buffer: match[2] ? Buffer.from(match[3], 'base64') : Buffer.from(decodeURIComponent(match[3]), 'utf8') }
|
|
338
|
+
} catch (_) {
|
|
339
|
+
return null
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
export function dataUriIsComplete(dataUri) {
|
|
344
|
+
const parsed = dataUriBuffer(dataUri)
|
|
345
|
+
return Boolean(parsed) && imageIsComplete(parsed.buffer, parsed.mime)
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
// Loopback, private, link-local, CGNAT and unspecified addresses.
|
|
349
|
+
const INTERNAL_ADDRESSES = new BlockList()
|
|
350
|
+
for (const [network, prefix] of [
|
|
351
|
+
['0.0.0.0', 8],
|
|
352
|
+
['10.0.0.0', 8],
|
|
353
|
+
['100.64.0.0', 10],
|
|
354
|
+
['127.0.0.0', 8],
|
|
355
|
+
['169.254.0.0', 16],
|
|
356
|
+
['172.16.0.0', 12],
|
|
357
|
+
['192.168.0.0', 16],
|
|
358
|
+
])
|
|
359
|
+
INTERNAL_ADDRESSES.addSubnet(network, prefix, 'ipv4')
|
|
360
|
+
for (const [network, prefix] of [
|
|
361
|
+
['::', 128],
|
|
362
|
+
['::1', 128],
|
|
363
|
+
['fc00::', 7],
|
|
364
|
+
['fe80::', 10],
|
|
365
|
+
])
|
|
366
|
+
INTERNAL_ADDRESSES.addSubnet(network, prefix, 'ipv6')
|
|
367
|
+
|
|
368
|
+
// The IPv4 address inside an IPv4-mapped IPv6 address (::ffff:a.b.c.d, which
|
|
369
|
+
// URL parsing writes as ::ffff:xxxx:yyyy), or null.
|
|
370
|
+
function mappedIPv4(address) {
|
|
371
|
+
let canonical
|
|
372
|
+
try {
|
|
373
|
+
canonical = new URL(`http://[${address}]/`).hostname.slice(1, -1)
|
|
374
|
+
} catch {
|
|
375
|
+
return null
|
|
376
|
+
}
|
|
377
|
+
const match = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(canonical)
|
|
378
|
+
if (!match) return null
|
|
379
|
+
const high = parseInt(match[1], 16)
|
|
380
|
+
const low = parseInt(match[2], 16)
|
|
381
|
+
return `${high >> 8}.${high & 0xff}.${low >> 8}.${low & 0xff}`
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// A URL hostname (IPv6 in brackets) that is loopback, private, link-local,
|
|
385
|
+
// CGNAT or an internal name. By name only: nothing is resolved. Trailing dots
|
|
386
|
+
// (`localhost.`) are ignored and IPv4-mapped IPv6 counts as its IPv4 address.
|
|
387
|
+
export function internalHostname(hostname) {
|
|
388
|
+
let host = String(hostname).toLowerCase().replace(/\.+$/, '')
|
|
389
|
+
if (host.startsWith('[') && host.endsWith(']')) host = host.slice(1, -1)
|
|
390
|
+
const family = isIP(host)
|
|
391
|
+
if (family === 6) {
|
|
392
|
+
const v4 = mappedIPv4(host)
|
|
393
|
+
return v4 ? INTERNAL_ADDRESSES.check(v4, 'ipv4') : INTERNAL_ADDRESSES.check(host, 'ipv6')
|
|
394
|
+
}
|
|
395
|
+
if (family === 4) return INTERNAL_ADDRESSES.check(host, 'ipv4')
|
|
396
|
+
return host === 'localhost' || /\.(internal|local|localhost|lan|home\.arpa|ts\.net)$/.test(host)
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
// ---- Links --------------------------------------------------------------------
|
|
400
|
+
// Links in the snapshot point at the capture base (often a local dev server).
|
|
401
|
+
// With a public URL configured they are rewritten to it; without one they stay
|
|
402
|
+
// as they are unless the capture base is itself a private host (links never
|
|
403
|
+
// navigate on the board anyway). Links to loopback, private, link-local, CGNAT
|
|
404
|
+
// and internal names are dropped, so no internal host leaks into a board you
|
|
405
|
+
// share.
|
|
406
|
+
export function createLinkRewriter({ from, to } = {}) {
|
|
407
|
+
let source = null
|
|
408
|
+
let target = null
|
|
409
|
+
try {
|
|
410
|
+
source = from ? new URL(from) : null
|
|
411
|
+
} catch {
|
|
412
|
+
source = null
|
|
413
|
+
}
|
|
414
|
+
try {
|
|
415
|
+
target = to ? new URL(to) : null
|
|
416
|
+
} catch {
|
|
417
|
+
target = null
|
|
418
|
+
}
|
|
419
|
+
const prefix = source ? source.pathname.replace(/\/+$/, '') : ''
|
|
420
|
+
const internal = (url) => internalHostname(url.hostname)
|
|
421
|
+
return (href) => {
|
|
422
|
+
let url
|
|
423
|
+
try {
|
|
424
|
+
url = new URL(href)
|
|
425
|
+
} catch {
|
|
426
|
+
return href
|
|
427
|
+
}
|
|
428
|
+
if (!/^https?:$/.test(url.protocol)) return href
|
|
429
|
+
if (source && url.origin === source.origin) {
|
|
430
|
+
if (!target) return internal(url) ? null : href
|
|
431
|
+
let pathname = url.pathname
|
|
432
|
+
if (prefix && (pathname === prefix || pathname.startsWith(`${prefix}/`))) pathname = pathname.slice(prefix.length) || '/'
|
|
433
|
+
return `${target.origin}${target.pathname.replace(/\/+$/, '')}${pathname}${url.search}${url.hash}`
|
|
434
|
+
}
|
|
435
|
+
return internal(url) ? null : href
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
// ---- Build --------------------------------------------------------------------
|
|
440
|
+
|
|
441
|
+
function walk(node, visit) {
|
|
442
|
+
visit(node)
|
|
443
|
+
for (const child of node.childNodes || []) walk(child, visit)
|
|
444
|
+
if (node.content) walk(node.content, visit)
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
function attr(node, name) {
|
|
448
|
+
return (node.attrs || []).find((item) => item.name === name)?.value
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
function setAttr(node, name, value) {
|
|
452
|
+
const existing = (node.attrs || []).find((item) => item.name === name)
|
|
453
|
+
if (existing) existing.value = value
|
|
454
|
+
else node.attrs.push({ name, value })
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// Builds the sanitized document. `getBody(url, maxBytes)` returns
|
|
458
|
+
// { buffer, mimeType, expectedLength } for a response the page loaded, or null.
|
|
459
|
+
export async function buildLiveHtml(snapshot, { getBody, links } = {}) {
|
|
460
|
+
const pageBase = snapshot.base
|
|
461
|
+
const cssImages = snapshot.cssImages || {}
|
|
462
|
+
const chunks = []
|
|
463
|
+
let cssTotal = 0
|
|
464
|
+
for (const chunk of snapshot.chunks || []) {
|
|
465
|
+
let text = chunk.text
|
|
466
|
+
if (text == null && chunk.href) {
|
|
467
|
+
const body = await getBody(stripHash(chunk.href), CSS_SHEET_MAX).catch(() => null)
|
|
468
|
+
text =
|
|
469
|
+
body && body.buffer.length <= CSS_SHEET_MAX && (!body.expectedLength || body.expectedLength === body.buffer.length)
|
|
470
|
+
? body.buffer.toString('utf8')
|
|
471
|
+
: ''
|
|
472
|
+
}
|
|
473
|
+
cssTotal += (text || '').length
|
|
474
|
+
if (cssTotal > CSS_MAX) throw new Error('the page CSS is over the cap')
|
|
475
|
+
chunks.push({ base: chunk.base || pageBase, text: text || '', media: chunk.media || '' })
|
|
476
|
+
}
|
|
477
|
+
const stats = { images: 0, imageBytes: 0, imagesDropped: 0, imagesRejected: 0, imagesShared: 0, fonts: 0, fontBytes: 0, fontsDropped: 0 }
|
|
478
|
+
|
|
479
|
+
// Pass 1: the document's images (already re-encoded data: URIs, or URLs
|
|
480
|
+
// the page could not read) and every remote URL the CSS references.
|
|
481
|
+
const tree = parse(String(snapshot.html || ''))
|
|
482
|
+
const imgNodes = []
|
|
483
|
+
walk(tree, (node) => {
|
|
484
|
+
if (node.tagName === 'img') imgNodes.push(node)
|
|
485
|
+
})
|
|
486
|
+
const wanted = []
|
|
487
|
+
const seen = new Set()
|
|
488
|
+
const collect = (url, kind) => {
|
|
489
|
+
const key = stripHash(url)
|
|
490
|
+
if (!seen.has(key)) {
|
|
491
|
+
seen.add(key)
|
|
492
|
+
wanted.push({ url: key, kind })
|
|
493
|
+
}
|
|
494
|
+
return null
|
|
495
|
+
}
|
|
496
|
+
for (const chunk of chunks) sanitizeCss(chunk.text, { base: chunk.base, resolveUrl: collect })
|
|
497
|
+
for (const node of imgNodes) {
|
|
498
|
+
const src = attr(node, 'src') || ''
|
|
499
|
+
if (src && !src.startsWith('data:')) {
|
|
500
|
+
try {
|
|
501
|
+
collect(new URL(src, pageBase).href, 'image')
|
|
502
|
+
} catch (_) {
|
|
503
|
+
/* placeholder later */
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
// Pass 2: everything the page could not re-encode comes from the bodies,
|
|
509
|
+
// only when complete. Re-encoded data from the page is checked too.
|
|
510
|
+
const inlined = new Map()
|
|
511
|
+
for (const item of wanted) {
|
|
512
|
+
if (item.kind === 'image' && cssImages[item.url]) {
|
|
513
|
+
if (dataUriIsComplete(cssImages[item.url])) inlined.set(item.url, cssImages[item.url])
|
|
514
|
+
else stats.imagesRejected += 1
|
|
515
|
+
continue
|
|
516
|
+
}
|
|
517
|
+
const body = await getBody(item.url, item.kind === 'font' ? FONT_MAX_EACH : BODY_MAX_EACH).catch(() => null)
|
|
518
|
+
const lengthOk = body && (!body.expectedLength || body.expectedLength === body.buffer.length)
|
|
519
|
+
const mime = String(body?.mimeType || '')
|
|
520
|
+
.split(';')[0]
|
|
521
|
+
.trim()
|
|
522
|
+
.toLowerCase()
|
|
523
|
+
if (item.kind === 'font') {
|
|
524
|
+
if (!lengthOk || !fontIsComplete(body.buffer)) {
|
|
525
|
+
stats.fontsDropped += 1
|
|
526
|
+
continue
|
|
527
|
+
}
|
|
528
|
+
const fontMime = /^(font\/|application\/(font-|x-font-|vnd\.ms-fontobject|octet-stream))/.test(mime) ? mime : 'font/woff2'
|
|
529
|
+
const fontUri = `data:${fontMime};base64,${body.buffer.toString('base64')}`
|
|
530
|
+
if (fontUri.length > FONT_MAX_EACH * 1.37 || stats.fontBytes + fontUri.length > FONT_BUDGET * 1.37) {
|
|
531
|
+
stats.fontsDropped += 1
|
|
532
|
+
continue
|
|
533
|
+
}
|
|
534
|
+
stats.fonts += 1
|
|
535
|
+
stats.fontBytes += fontUri.length
|
|
536
|
+
inlined.set(item.url, fontUri)
|
|
537
|
+
continue
|
|
538
|
+
}
|
|
539
|
+
if (!body || !mime.startsWith('image/')) {
|
|
540
|
+
stats.imagesDropped += 1
|
|
541
|
+
continue
|
|
542
|
+
}
|
|
543
|
+
if (!lengthOk || !imageIsComplete(body.buffer, mime)) {
|
|
544
|
+
stats.imagesRejected += 1
|
|
545
|
+
continue
|
|
546
|
+
}
|
|
547
|
+
inlined.set(item.url, `data:${mime};base64,${body.buffer.toString('base64')}`)
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
// Pass 3: the budget counts each distinct image once (identical images are
|
|
551
|
+
// stored once below). Over budget, the largest go first.
|
|
552
|
+
const distinct = new Map() // data URI -> size
|
|
553
|
+
for (const node of imgNodes) {
|
|
554
|
+
let src = attr(node, 'src') || ''
|
|
555
|
+
if (src && !src.startsWith('data:')) {
|
|
556
|
+
try {
|
|
557
|
+
src = inlined.get(stripHash(new URL(src, pageBase).href)) || ''
|
|
558
|
+
} catch (_) {
|
|
559
|
+
src = ''
|
|
560
|
+
}
|
|
561
|
+
setAttr(node, 'src', src)
|
|
562
|
+
}
|
|
563
|
+
if (src.startsWith('data:') && src !== TRANSPARENT_GIF) {
|
|
564
|
+
if (!dataUriIsComplete(src)) {
|
|
565
|
+
stats.imagesRejected += 1
|
|
566
|
+
setAttr(node, 'src', '')
|
|
567
|
+
continue
|
|
568
|
+
}
|
|
569
|
+
distinct.set(src, src.length)
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
for (const value of inlined.values()) if (value.startsWith('data:image/')) distinct.set(value, value.length)
|
|
573
|
+
let total = [...distinct.values()].reduce((sum, size) => sum + size, 0)
|
|
574
|
+
const dropped = new Set()
|
|
575
|
+
for (const [value, size] of [...distinct.entries()].sort((a, b) => b[1] - a[1])) {
|
|
576
|
+
if (total <= IMAGE_BUDGET) break
|
|
577
|
+
dropped.add(value)
|
|
578
|
+
total -= size
|
|
579
|
+
stats.imagesDropped += 1
|
|
580
|
+
}
|
|
581
|
+
for (const node of imgNodes) {
|
|
582
|
+
const src = attr(node, 'src') || ''
|
|
583
|
+
if (!src || dropped.has(src)) setAttr(node, 'src', placeholderSvg(attr(node, 'data-sw-w'), attr(node, 'data-sw-h')))
|
|
584
|
+
}
|
|
585
|
+
for (const [url, value] of [...inlined]) if (dropped.has(value)) inlined.delete(url)
|
|
586
|
+
stats.images = distinct.size - dropped.size
|
|
587
|
+
stats.imageBytes = total
|
|
588
|
+
const lookup = (url) => inlined.get(stripHash(url)) || null
|
|
589
|
+
|
|
590
|
+
// Pass 4: one combined stylesheet, then the strict sanitize of the whole
|
|
591
|
+
// document. Media queries on the color scheme (and reduced motion) are
|
|
592
|
+
// resolved to what the capture saw, and the root's color-scheme is pinned.
|
|
593
|
+
const scheme = snapshotScheme(snapshot)
|
|
594
|
+
const css = chunks
|
|
595
|
+
.map((chunk) => resolveMediaInCss(wrapMedia(sanitizeCss(chunk.text, { base: chunk.base, resolveUrl: lookup }), chunk.media), scheme))
|
|
596
|
+
.concat(FRAME_RULES, `:root{color-scheme:only ${pinnedColorScheme(scheme.root, scheme.colorScheme)}!important}`)
|
|
597
|
+
.join('\n')
|
|
598
|
+
.replace(/<\/(style)/gi, '<\\/$1')
|
|
599
|
+
const prepared = serialize(tree)
|
|
600
|
+
// Function replacements: page CSS may contain $& or $' and must stay literal.
|
|
601
|
+
const withStyle = prepared.includes('</head>')
|
|
602
|
+
? prepared.replace('</head>', () => `<style>${css}</style></head>`)
|
|
603
|
+
: prepared.replace(/<html[^>]*>/i, (open) => `${open}<head><style>${css}</style></head>`)
|
|
604
|
+
const sanitized = sanitizeDocument(withStyle, {
|
|
605
|
+
base: pageBase,
|
|
606
|
+
resolveUrl: lookup,
|
|
607
|
+
rewriteHref: links ? createLinkRewriter(links) : null,
|
|
608
|
+
})
|
|
609
|
+
const html = shareRepeatedImages(sanitized, stats)
|
|
610
|
+
return { html, stats, colorScheme: scheme.colorScheme }
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
// Pass 5: an image used more than once is stored once, as a CSS rule
|
|
614
|
+
// (`content: url(...)` on the img, which keeps its intrinsic size), and each
|
|
615
|
+
// use points at it with a class. Applied after the sanitizer, to its output.
|
|
616
|
+
function shareRepeatedImages(html, stats) {
|
|
617
|
+
const tree = parse(html)
|
|
618
|
+
const uses = new Map()
|
|
619
|
+
walk(tree, (node) => {
|
|
620
|
+
if (node.tagName !== 'img') return
|
|
621
|
+
const src = attr(node, 'src') || ''
|
|
622
|
+
// Only plain base64 URIs are moved into CSS: their alphabet cannot close
|
|
623
|
+
// the string, the url() or the style element.
|
|
624
|
+
if (BASE64_IMAGE_RE.test(src) && src.length > 2048) uses.set(src, (uses.get(src) || 0) + 1)
|
|
625
|
+
})
|
|
626
|
+
const shared = new Map()
|
|
627
|
+
for (const [src, count] of uses) if (count > 1) shared.set(src, `sw-img-${shared.size + 1}`)
|
|
628
|
+
if (!shared.size) return html
|
|
629
|
+
let head = null
|
|
630
|
+
walk(tree, (node) => {
|
|
631
|
+
if (node.tagName === 'head' && !head) head = node
|
|
632
|
+
if (node.tagName !== 'img') return
|
|
633
|
+
const className = shared.get(attr(node, 'src') || '')
|
|
634
|
+
if (!className) return
|
|
635
|
+
setAttr(node, 'src', TRANSPARENT_GIF)
|
|
636
|
+
const existing = attr(node, 'class')
|
|
637
|
+
setAttr(node, 'class', existing ? `${existing} ${className}` : className)
|
|
638
|
+
stats.imagesShared += 1
|
|
639
|
+
})
|
|
640
|
+
const rules = [...shared].map(([src, className]) => `img.${className}{content:url("${src}")}`).join('\n')
|
|
641
|
+
if (head) {
|
|
642
|
+
const style = {
|
|
643
|
+
nodeName: 'style',
|
|
644
|
+
tagName: 'style',
|
|
645
|
+
attrs: [],
|
|
646
|
+
namespaceURI: 'http://www.w3.org/1999/xhtml',
|
|
647
|
+
childNodes: [],
|
|
648
|
+
parentNode: head,
|
|
649
|
+
}
|
|
650
|
+
style.childNodes.push({ nodeName: '#text', value: rules, parentNode: style })
|
|
651
|
+
head.childNodes.push(style)
|
|
652
|
+
}
|
|
653
|
+
return serialize(tree)
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
// The fourth lock: the document carries its own CSP, so even a copy opened
|
|
657
|
+
// outside Shapewall (a static export, a file on disk) cannot run script or
|
|
658
|
+
// load anything remote.
|
|
659
|
+
export const LIVE_CSP = "default-src 'none'; img-src data:; style-src 'unsafe-inline'; font-src data:; base-uri 'none'; form-action 'none'"
|
|
660
|
+
export function withCspMeta(html) {
|
|
661
|
+
const meta = `<meta http-equiv="Content-Security-Policy" content="${LIVE_CSP}">`
|
|
662
|
+
return /<head>/i.test(html) ? html.replace(/<head>/i, () => `<head>${meta}`) : `${meta}${html}`
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
// Did this response, or any redirect hop that led to it, come from an
|
|
666
|
+
// internal host? At most 20 hops (browsers stop there too).
|
|
667
|
+
export async function responseTouchesInternal(response, isInternal) {
|
|
668
|
+
try {
|
|
669
|
+
if (await isInternal(response.url())) return true
|
|
670
|
+
let request = response.request?.() || null
|
|
671
|
+
for (let hops = 0; request && hops < 21; hops += 1) {
|
|
672
|
+
if (await isInternal(request.url())) return true
|
|
673
|
+
request = request.redirectedFrom?.() || null
|
|
674
|
+
}
|
|
675
|
+
} catch {
|
|
676
|
+
// A response we cannot place is not inlined.
|
|
677
|
+
return true
|
|
678
|
+
}
|
|
679
|
+
return false
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
// Snapshot a Playwright page and write the sanitized document. `responses`
|
|
683
|
+
// maps URL (without hash) -> Playwright Response, recorded while the page
|
|
684
|
+
// loaded. `isCancelled()` is true once the shot has timed out; then nothing is
|
|
685
|
+
// written. `isInternal(url)` is the private-host guard's answer (requests.js):
|
|
686
|
+
// a response that came from an internal host, or through a redirect chain
|
|
687
|
+
// that touched one, is never inlined (the browser does not route redirect
|
|
688
|
+
// hops, so the guard cannot stop those itself).
|
|
689
|
+
export async function writeLiveHtml({ page, responses, file, links, isCancelled = () => false, size = null, isInternal = null }) {
|
|
690
|
+
const value = await page.evaluate(SNAPSHOT_EXPRESSION)
|
|
691
|
+
if (value?.tooLarge) throw new Error(value.tooLarge)
|
|
692
|
+
if (!value?.html) throw new Error('the page could not be snapshotted')
|
|
693
|
+
const getBody = async (url, maxBytes) => {
|
|
694
|
+
const response = responses.get(stripHash(url))
|
|
695
|
+
// Only whole 200 responses.
|
|
696
|
+
if (!response || isCancelled() || response.status() !== 200) return null
|
|
697
|
+
if (isInternal && (await responseTouchesInternal(response, isInternal))) return null
|
|
698
|
+
const headers = await response.allHeaders().catch(() => ({}))
|
|
699
|
+
const declared = headers['content-length'] ? Number(headers['content-length']) : null
|
|
700
|
+
if (maxBytes && Number.isFinite(declared) && declared > maxBytes) return null
|
|
701
|
+
let timer = null
|
|
702
|
+
let buffer
|
|
703
|
+
try {
|
|
704
|
+
buffer = await Promise.race([
|
|
705
|
+
response.body().catch(() => null),
|
|
706
|
+
new Promise((resolve) => {
|
|
707
|
+
timer = setTimeout(() => resolve(null), 5000)
|
|
708
|
+
}),
|
|
709
|
+
])
|
|
710
|
+
} finally {
|
|
711
|
+
// One timer per asset: not left running 5 s after the body came.
|
|
712
|
+
clearTimeout(timer)
|
|
713
|
+
}
|
|
714
|
+
if (!buffer || (maxBytes && buffer.length > maxBytes)) return null
|
|
715
|
+
// Content-Length describes the body only when it was not compressed.
|
|
716
|
+
const encoding = String(headers['content-encoding'] || '')
|
|
717
|
+
.trim()
|
|
718
|
+
.toLowerCase()
|
|
719
|
+
const expectedLength = (!encoding || encoding === 'identity') && Number.isFinite(declared) ? declared : null
|
|
720
|
+
const mimeType = String(headers['content-type'] || '')
|
|
721
|
+
.split(';')[0]
|
|
722
|
+
.trim()
|
|
723
|
+
return { buffer, mimeType, expectedLength }
|
|
724
|
+
}
|
|
725
|
+
const built = await buildLiveHtml(value, { getBody, links })
|
|
726
|
+
const html = withCspMeta(built.html)
|
|
727
|
+
const bytes = Buffer.byteLength(html)
|
|
728
|
+
if (bytes > FILE_MAX) throw new Error(`live HTML is ${(bytes / 1048576).toFixed(1)} MB, over the ${FILE_MAX / 1048576} MB cap`)
|
|
729
|
+
if (isCancelled()) throw new Error('the shot timed out before the live HTML was written')
|
|
730
|
+
// The element map comes from the document as written so far; then its
|
|
731
|
+
// controls are made inert for the board (see INERT_RULES).
|
|
732
|
+
const mapFile = elementMapFile(file)
|
|
733
|
+
let elements = null
|
|
734
|
+
if (mapFile) {
|
|
735
|
+
try {
|
|
736
|
+
const map = await buildElementMapFor({
|
|
737
|
+
browser: page.context().browser(),
|
|
738
|
+
html,
|
|
739
|
+
size: size ? { width: size.width, height: size.height } : page.viewportSize(),
|
|
740
|
+
isCancelled,
|
|
741
|
+
})
|
|
742
|
+
const text = JSON.stringify(map)
|
|
743
|
+
if (text.length > ELEMENT_MAP_LIMITS.bytes) throw new Error('the element map is over its cap')
|
|
744
|
+
elements = { map, text }
|
|
745
|
+
} catch (error) {
|
|
746
|
+
elements = { error: String(error?.message || error).split('\n')[0] }
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
const written = elements?.map ? withInertControls(html) : html
|
|
750
|
+
if (isCancelled()) throw new Error('the shot timed out before the live HTML was written')
|
|
751
|
+
writeFileAtomic(file, written)
|
|
752
|
+
if (mapFile) {
|
|
753
|
+
if (elements?.map) writeFileAtomic(mapFile, elements.text)
|
|
754
|
+
// A map from an older capture must not sit beside the new document.
|
|
755
|
+
else fs.rmSync(mapFile, { force: true })
|
|
756
|
+
}
|
|
757
|
+
return {
|
|
758
|
+
bytes: Buffer.byteLength(written),
|
|
759
|
+
sha256: crypto.createHash('sha256').update(written).digest('hex'),
|
|
760
|
+
stats: built.stats,
|
|
761
|
+
colorScheme: built.colorScheme,
|
|
762
|
+
elements: elements?.map
|
|
763
|
+
? { status: 'ok', count: elements.map.elements.length, ...(elements.map.truncated ? { truncated: true } : {}) }
|
|
764
|
+
: elements
|
|
765
|
+
? { status: 'failed', error: elements.error.slice(0, 200) }
|
|
766
|
+
: null,
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
// ---- Element maps ---------------------------------------------------------------
|
|
771
|
+
// The map (element-map.js) is measured on the written document rendered the
|
|
772
|
+
// way the board renders it: a plain page (no mobile emulation: the board's
|
|
773
|
+
// frame is an iframe of the capture's native size), device scale 1, nothing
|
|
774
|
+
// else loaded. The document allows no script (its CSP); the measuring runs
|
|
775
|
+
// through the DevTools protocol.
|
|
776
|
+
|
|
777
|
+
export const elementMapFile = (htmlFile) =>
|
|
778
|
+
String(htmlFile).endsWith('.html') ? String(htmlFile).replace(/\.html$/, '.elements.json') : null
|
|
779
|
+
|
|
780
|
+
const MAP_TIMEOUT_MS = 20000
|
|
781
|
+
|
|
782
|
+
export async function buildElementMapFor({ browser, html, size, isCancelled = () => false }) {
|
|
783
|
+
if (!browser) throw new Error('no browser for the element map')
|
|
784
|
+
if (!size?.width || !size?.height) throw new Error('no viewport size for the element map')
|
|
785
|
+
const context = await browser.newContext({
|
|
786
|
+
viewport: { width: size.width, height: size.height },
|
|
787
|
+
deviceScaleFactor: 1,
|
|
788
|
+
isMobile: false,
|
|
789
|
+
hasTouch: false,
|
|
790
|
+
// Nothing leaves the page: the document is self-contained.
|
|
791
|
+
offline: true,
|
|
792
|
+
})
|
|
793
|
+
let timer = null
|
|
794
|
+
try {
|
|
795
|
+
const work = (async () => {
|
|
796
|
+
const page = await context.newPage()
|
|
797
|
+
await page.setContent(html, { waitUntil: 'load', timeout: MAP_TIMEOUT_MS })
|
|
798
|
+
// Fonts and images (all data: URIs) decoded, as on the board once loaded.
|
|
799
|
+
await page.evaluate(async () => {
|
|
800
|
+
await Promise.all(Array.from(document.images).map((image) => (image.decode ? image.decode().catch(() => {}) : null)))
|
|
801
|
+
if (document.fonts?.ready) await document.fonts.ready
|
|
802
|
+
})
|
|
803
|
+
if (isCancelled()) throw new Error('cancelled')
|
|
804
|
+
return cleanElementMap(await page.evaluate(ELEMENT_MAP_EXPRESSION))
|
|
805
|
+
})()
|
|
806
|
+
const timeout = new Promise((resolve, reject) => {
|
|
807
|
+
timer = setTimeout(() => reject(new Error('the element map timed out')), MAP_TIMEOUT_MS + 5000)
|
|
808
|
+
})
|
|
809
|
+
return await Promise.race([work, timeout])
|
|
810
|
+
} finally {
|
|
811
|
+
clearTimeout(timer)
|
|
812
|
+
await context.close().catch(() => {})
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
// With an element map the board never touches the frame's DOM, so it can
|
|
817
|
+
// no longer stop a click on a link (which would navigate the frame) or a
|
|
818
|
+
// summary (which would change the layout under the map) while text is being
|
|
819
|
+
// selected. These controls take no pointer events instead; the text still
|
|
820
|
+
// selects. Only in documents that have a map: without one the local board
|
|
821
|
+
// reads the DOM and needs them hit-testable.
|
|
822
|
+
export const INERT_RULES =
|
|
823
|
+
'a[href],area[href],button,input,select,textarea,label,summary,[role=button],[role=link],[role=tab],[role=menuitem]{pointer-events:none!important}'
|
|
824
|
+
export function withInertControls(html) {
|
|
825
|
+
const style = `<style>${INERT_RULES}</style>`
|
|
826
|
+
return /<\/head>/i.test(html) ? html.replace(/<\/head>/i, () => `${style}</head>`) : `${style}${html}`
|
|
827
|
+
}
|