shapewall 0.0.0-stage → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +139 -0
- package/Dockerfile.agent +45 -0
- package/LICENSE +202 -0
- package/NOTICE +12 -0
- package/README.md +651 -2
- package/THIRD_PARTY.md +165 -0
- package/bin/shapewall.js +29 -0
- package/dist/agent/package-lock.json +8384 -0
- package/dist/client/assets/BoardChat-D5SFd8HE.js +28 -0
- package/dist/client/assets/BuzzDialog-HWXBy52W.js +1 -0
- package/dist/client/assets/ChecksPanel-DTdImWoD.js +1 -0
- package/dist/client/assets/History-CEv6b-av.js +1 -0
- package/dist/client/assets/LocalSettings-C4vriEVQ.js +1 -0
- package/dist/client/assets/Management-DCfhcEUY.js +1 -0
- package/dist/client/assets/QuickConnect-DQGgUsl1.js +1 -0
- package/dist/client/assets/ShareDialog-C_plIto0.js +2 -0
- package/dist/client/assets/app-update-dialog-2hPz_N9G.js +1 -0
- package/dist/client/assets/circle-x-BZMPi8Zf.js +1 -0
- package/dist/client/assets/index-ByQRKVit.css +2 -0
- package/dist/client/assets/index-DweQ8Qtu.js +29 -0
- package/dist/client/assets/refresh-cw-CRow0Nbq.js +1 -0
- package/dist/client/assets/runtime-Bft6x_6E.js +4 -0
- package/dist/client/assets/square-A8EnIYLR.js +1 -0
- package/dist/client/assets/tts-dist-DYlBQmwG.js +18 -0
- package/dist/client/assets/tts-en-g2p-BPUtBFGG.js +1 -0
- package/dist/client/assets/tts-ort-wasm-simd-threaded-qh98TCA_.wasm +0 -0
- package/dist/client/assets/tts-ort.wasm.bundle.min-rEkcs_ow.js +12 -0
- package/dist/client/assets/tts-rolldown-runtime-D8NudN84.js +1 -0
- package/dist/client/assets/tts-spacy-en-tokenizer-6SNMl--A.json +1 -0
- package/dist/client/assets/tts-tts.worker-Bzm6aWbv.js +1 -0
- package/dist/client/assets/tts-us_gold-DcjeNMH1.json +92589 -0
- package/dist/client/assets/tts-us_silver-C964mEB5.json +93363 -0
- package/dist/client/assets/ui-CLbws0rx.js +41 -0
- package/dist/client/build-info.json +1 -0
- package/dist/client/favicon.svg +7 -0
- package/dist/client/icons/apple-touch-icon.png +0 -0
- package/dist/client/icons/icon-192.png +0 -0
- package/dist/client/icons/icon-512.png +0 -0
- package/dist/client/icons/maskable-512.png +0 -0
- package/dist/client/index.html +25 -0
- package/dist/client/manifest.webmanifest +15 -0
- package/dist/client/sw.js +53 -0
- package/dist/client/theme-boot.js +27 -0
- package/dist/console/assets/LocalAccountsApp-BQCcmj3x.js +1 -0
- package/dist/console/assets/LocalConsole-Bb2FUDs7.js +1 -0
- package/dist/console/assets/app-update-dialog-DIeYepLZ.js +1 -0
- package/dist/console/assets/console-C7uJLLOn.js +51 -0
- package/dist/console/assets/console-R8IBY3Am.css +2 -0
- package/dist/console/build-info.json +1 -0
- package/dist/console/console.html +21 -0
- package/dist/console/favicon.svg +7 -0
- package/dist/console/icons/apple-touch-icon.png +0 -0
- package/dist/console/icons/icon-192.png +0 -0
- package/dist/console/icons/icon-512.png +0 -0
- package/dist/console/icons/maskable-512.png +0 -0
- package/dist/console/manifest.webmanifest +15 -0
- package/dist/console/theme-boot.js +27 -0
- package/docs/agent-hardening.md +342 -0
- package/docs/agent-recipes.md +187 -0
- package/docs/agent-workflow.md +320 -0
- package/docs/agents.md +894 -0
- package/docs/architecture.md +242 -0
- package/docs/buzz.md +140 -0
- package/docs/capture-warnings.md +57 -0
- package/docs/config.md +489 -0
- package/docs/deployment.md +289 -0
- package/docs/design-system.md +162 -0
- package/docs/github.md +107 -0
- package/docs/hosted.md +764 -0
- package/docs/images/board-overview.png +0 -0
- package/docs/images/journeys.png +0 -0
- package/docs/images/live-comment.png +0 -0
- package/docs/images/phone-comment.png +0 -0
- package/docs/images/phone-overview.png +0 -0
- package/docs/images/variant.png +0 -0
- package/docs/interaction-audits.md +40 -0
- package/docs/journeys.md +137 -0
- package/docs/local-accounts.md +80 -0
- package/docs/notifications.md +237 -0
- package/docs/page-checks.md +69 -0
- package/docs/performance.md +174 -0
- package/docs/starter-agent.md +325 -0
- package/docs/updates.md +87 -0
- package/examples/audits/interaction.audit.json +17 -0
- package/package.json +132 -4
- package/src/accounts/avatar.js +121 -0
- package/src/accounts/local.js +464 -0
- package/src/agent/acp/command.js +204 -0
- package/src/agent/acp/detect.js +75 -0
- package/src/agent/acp/sandbox.js +160 -0
- package/src/agent/agent-health.js +75 -0
- package/src/agent/backends/acp.js +1297 -0
- package/src/agent/backends/index.js +61 -0
- package/src/agent/backends/openai.js +308 -0
- package/src/agent/composer.js +221 -0
- package/src/agent/config.js +338 -0
- package/src/agent/context.js +191 -0
- package/src/agent/control.js +122 -0
- package/src/agent/errors.js +17 -0
- package/src/agent/feedback-loop.js +137 -0
- package/src/agent/manage.js +94 -0
- package/src/agent/next-steps.js +51 -0
- package/src/agent/owner-mcp.js +38 -0
- package/src/agent/plain-text.js +52 -0
- package/src/agent/ratings-core.js +213 -0
- package/src/agent/ratings.js +100 -0
- package/src/agent/reply-text.js +67 -0
- package/src/agent/routes.js +1065 -0
- package/src/agent/run.js +326 -0
- package/src/agent/settings-routes.js +704 -0
- package/src/agent/showable.js +23 -0
- package/src/agent/starter/approvals.js +181 -0
- package/src/agent/starter/build-image.js +90 -0
- package/src/agent/starter/cli.js +163 -0
- package/src/agent/starter/disk.js +53 -0
- package/src/agent/starter/doctor.js +180 -0
- package/src/agent/starter/egress-proxy.js +351 -0
- package/src/agent/starter/gate.js +90 -0
- package/src/agent/starter/loop.js +114 -0
- package/src/agent/starter/model-relay.js +289 -0
- package/src/agent/starter/pause.js +14 -0
- package/src/agent/starter/probe.js +272 -0
- package/src/agent/starter/prompt.js +41 -0
- package/src/agent/starter/push-guard.js +146 -0
- package/src/agent/starter/rpc.js +164 -0
- package/src/agent/starter/runtime.js +287 -0
- package/src/agent/starter/sandbox-main.js +121 -0
- package/src/agent/starter/settings.js +290 -0
- package/src/agent/starter/source.js +69 -0
- package/src/agent/starter/supervisor.js +755 -0
- package/src/agent/starter/sweep.js +46 -0
- package/src/agent/starter/tools.js +524 -0
- package/src/agent/starter/watch.js +84 -0
- package/src/agent/starter/worktree.js +409 -0
- package/src/agent/text-diff.js +210 -0
- package/src/agent/thread-history.js +223 -0
- package/src/agent/threads.js +399 -0
- package/src/agent/tools.js +200 -0
- package/src/agent/turn-changes.js +424 -0
- package/src/agent/undo-state.js +26 -0
- package/src/agent/user-settings.js +308 -0
- package/src/auth/record.js +375 -0
- package/src/broken-links.js +164 -0
- package/src/buzz/client.js +75 -0
- package/src/buzz/config.js +151 -0
- package/src/buzz/environment.js +13 -0
- package/src/buzz/gateway.js +192 -0
- package/src/buzz/routes.js +61 -0
- package/src/buzz/signer-child.js +205 -0
- package/src/buzz/store.js +89 -0
- package/src/buzz/transport.js +60 -0
- package/src/buzz/verify.js +140 -0
- package/src/capture/actions.js +159 -0
- package/src/capture/capture.js +776 -0
- package/src/capture/dom-names.js +189 -0
- package/src/capture/element-map.js +509 -0
- package/src/capture/live.js +827 -0
- package/src/capture/media-scheme.js +317 -0
- package/src/capture/network-summary.js +86 -0
- package/src/capture/pacing.js +34 -0
- package/src/capture/requests.js +167 -0
- package/src/capture/sanitize.js +752 -0
- package/src/checks/accessibility.js +134 -0
- package/src/checks/budgets.js +58 -0
- package/src/checks/cli.js +55 -0
- package/src/checks/comparison.js +113 -0
- package/src/checks/environments.js +32 -0
- package/src/checks/flow-config.js +82 -0
- package/src/checks/flow-origin.js +53 -0
- package/src/checks/flows.js +398 -0
- package/src/checks/lighthouse-auth.js +145 -0
- package/src/checks/lighthouse-findings.js +130 -0
- package/src/checks/lighthouse.js +395 -0
- package/src/checks/robots.js +85 -0
- package/src/checks/routes.js +52 -0
- package/src/checks/run-control.js +259 -0
- package/src/checks/runner.js +724 -0
- package/src/checks/schema.js +133 -0
- package/src/checks/seo-analysis.js +443 -0
- package/src/checks/seo-probe.js +190 -0
- package/src/checks/seo-snapshot.js +99 -0
- package/src/checks/store.js +214 -0
- package/src/checks/transport.js +47 -0
- package/src/cli/args.js +319 -0
- package/src/cli/capture.js +79 -0
- package/src/cli/commands.js +92 -0
- package/src/cli/comments.js +84 -0
- package/src/cli/help.js +243 -0
- package/src/cli/init.js +214 -0
- package/src/cli/misc.js +64 -0
- package/src/cli/notify.js +87 -0
- package/src/cli/open.js +22 -0
- package/src/cli/serve.js +274 -0
- package/src/cli/variant.js +82 -0
- package/src/cli/watch.js +70 -0
- package/src/cli.js +130 -0
- package/src/comment-fields.js +76 -0
- package/src/comments-cli.js +208 -0
- package/src/config-reload.js +100 -0
- package/src/config.js +1061 -0
- package/src/devices-cli.js +21 -0
- package/src/diff.js +95 -0
- package/src/discover.js +284 -0
- package/src/doctor.js +295 -0
- package/src/event-log.js +268 -0
- package/src/events-cli.js +102 -0
- package/src/events.js +113 -0
- package/src/export.js +119 -0
- package/src/feedback.js +246 -0
- package/src/github.js +551 -0
- package/src/hosted/LICENSE.md +105 -0
- package/src/hosted/access-cli.js +78 -0
- package/src/hosted/admin.js +682 -0
- package/src/hosted/agent-approvals.js +163 -0
- package/src/hosted/agent-policy.js +189 -0
- package/src/hosted/app.js +510 -0
- package/src/hosted/auth-routes.js +540 -0
- package/src/hosted/auth.js +536 -0
- package/src/hosted/backup.js +483 -0
- package/src/hosted/board-context.js +35 -0
- package/src/hosted/board-routes.js +486 -0
- package/src/hosted/board-view.js +94 -0
- package/src/hosted/clean-worker.js +24 -0
- package/src/hosted/cleaner.js +89 -0
- package/src/hosted/comments.js +355 -0
- package/src/hosted/console-routes.js +401 -0
- package/src/hosted/cookies.js +18 -0
- package/src/hosted/crypto.js +80 -0
- package/src/hosted/db.js +496 -0
- package/src/hosted/device-routes.js +175 -0
- package/src/hosted/device.js +233 -0
- package/src/hosted/events.js +63 -0
- package/src/hosted/guards.js +172 -0
- package/src/hosted/guest-view.js +65 -0
- package/src/hosted/http.js +39 -0
- package/src/hosted/import.js +300 -0
- package/src/hosted/mail.js +73 -0
- package/src/hosted/mcp-http.js +71 -0
- package/src/hosted/membership.js +114 -0
- package/src/hosted/operator.js +478 -0
- package/src/hosted/ops-cli.js +345 -0
- package/src/hosted/pages.js +146 -0
- package/src/hosted/permissions.js +113 -0
- package/src/hosted/presence.js +372 -0
- package/src/hosted/projects.js +124 -0
- package/src/hosted/public.js +109 -0
- package/src/hosted/push.js +622 -0
- package/src/hosted/quotas.js +115 -0
- package/src/hosted/ratelimit.js +96 -0
- package/src/hosted/rebind.js +316 -0
- package/src/hosted/retention.js +128 -0
- package/src/hosted/server.js +260 -0
- package/src/hosted/setup-link.js +64 -0
- package/src/hosted/setup.js +323 -0
- package/src/hosted/shares.js +444 -0
- package/src/hosted/tailscale.js +204 -0
- package/src/hosted/tenancy.js +301 -0
- package/src/hosted/token-api.js +455 -0
- package/src/hosted/token-events.js +60 -0
- package/src/hosted/tokens.js +57 -0
- package/src/hosted/turnstile.js +49 -0
- package/src/hosted/usercontent.js +154 -0
- package/src/hosted/zip.js +155 -0
- package/src/invocation.js +58 -0
- package/src/journey-paths.js +282 -0
- package/src/journeys.js +189 -0
- package/src/links.js +106 -0
- package/src/local-projects.js +182 -0
- package/src/lock.js +193 -0
- package/src/mcp.js +417 -0
- package/src/mentions.js +151 -0
- package/src/net.js +15 -0
- package/src/notify/config.js +275 -0
- package/src/notify/index.js +301 -0
- package/src/notify/lease.js +73 -0
- package/src/notify/messages.js +212 -0
- package/src/notify/run.js +174 -0
- package/src/pixeldiff.js +49 -0
- package/src/png.js +143 -0
- package/src/reactions.js +111 -0
- package/src/remote/cli.js +133 -0
- package/src/remote/credentials.js +62 -0
- package/src/remote/ids.js +12 -0
- package/src/remote/login.js +155 -0
- package/src/remote/remote-mcp.js +237 -0
- package/src/remote/remote-watch.js +291 -0
- package/src/remote/remote.js +313 -0
- package/src/remote/share-state.js +78 -0
- package/src/remote/share.js +369 -0
- package/src/remote/terminal.js +13 -0
- package/src/serve-state.js +64 -0
- package/src/server/app.js +824 -0
- package/src/server/devices.js +186 -0
- package/src/server/local-console.js +28 -0
- package/src/server/local-user.js +59 -0
- package/src/server/pairing-routes.js +159 -0
- package/src/server/share-routes.js +89 -0
- package/src/server/tailscale.js +97 -0
- package/src/sitemap.js +431 -0
- package/src/store.js +983 -0
- package/src/tokens.js +11 -0
- package/src/update/routes.js +245 -0
- package/src/watch.js +403 -0
- package/src/which.js +58 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Devices } from './server/devices.js'
|
|
2
|
+
export function devicesCommand(config, positional, flags, out, env = process.env) {
|
|
3
|
+
const devices = new Devices(config.dir, { env })
|
|
4
|
+
const [action = 'list', id] = positional
|
|
5
|
+
if (action === 'list') {
|
|
6
|
+
const list = devices.list()
|
|
7
|
+
if (flags.json) out(JSON.stringify(list, null, 2))
|
|
8
|
+
else {
|
|
9
|
+
for (const item of list.devices) out(`${item.id} ${item.name} (last seen ${new Date(item.lastSeenAt).toISOString()})`)
|
|
10
|
+
for (const item of list.pending) out(`Pending ${item.code.slice(0, 4)}-${item.code.slice(4)} ${item.name} (${item.ip})`)
|
|
11
|
+
if (!list.devices.length && !list.pending.length) out('No paired devices or pending requests for this project.')
|
|
12
|
+
}
|
|
13
|
+
} else if (['approve', 'deny'].includes(action)) {
|
|
14
|
+
if (!id) throw new Error(`devices ${action} needs the code shown on the device.`)
|
|
15
|
+
const result = devices.decide(id, action === 'approve')
|
|
16
|
+
out(`${result.name}: ${result.state}`)
|
|
17
|
+
} else if (action === 'revoke') {
|
|
18
|
+
if (!id && !flags.all) throw new Error('devices revoke needs a device id or --all.')
|
|
19
|
+
out(`Revoked ${devices.revoke(flags.all ? '--all' : id)} device(s).`)
|
|
20
|
+
} else throw new Error('devices: use list, approve CODE, deny CODE, or revoke ID|--all.')
|
|
21
|
+
}
|
package/src/diff.js
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
// Pixel diffs between two environments' captures of the same page and
|
|
2
|
+
// viewport (pixelmatch). Computed when the board asks for a comparison (and
|
|
3
|
+
// after an environment capture), then cached under
|
|
4
|
+
// <board>/diffs/<a>_<b>/: a transparent PNG with the changed pixels in red,
|
|
5
|
+
// to lay over the screenshot, and index.json with the changed share. A cached
|
|
6
|
+
// entry is reused while both source files are unchanged (size and mtime).
|
|
7
|
+
|
|
8
|
+
import fs from 'node:fs'
|
|
9
|
+
import path from 'node:path'
|
|
10
|
+
import { PNG } from 'pngjs'
|
|
11
|
+
import { withFileLock } from './lock.js'
|
|
12
|
+
import { DIFF_THRESHOLD, diffPixels } from './pixeldiff.js'
|
|
13
|
+
import { capturesDir, diffsDir, readJson, writeFileAtomic, writeJsonAtomic } from './store.js'
|
|
14
|
+
|
|
15
|
+
const CACHE_FORMAT = 1
|
|
16
|
+
|
|
17
|
+
// Compares two PNG buffers (see diffPixels). Returns { width, height,
|
|
18
|
+
// changed, total, percent, png } where png is the diff mask (red where
|
|
19
|
+
// changed, transparent elsewhere).
|
|
20
|
+
export function diffPngs(bufferA, bufferB, { threshold = DIFF_THRESHOLD } = {}) {
|
|
21
|
+
const { data, ...result } = diffPixels(PNG.sync.read(bufferA), PNG.sync.read(bufferB), { threshold })
|
|
22
|
+
const mask = new PNG({ width: result.width, height: result.height })
|
|
23
|
+
mask.data.set(data)
|
|
24
|
+
return { ...result, png: PNG.sync.write(mask) }
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function stamp(file) {
|
|
28
|
+
try {
|
|
29
|
+
const stat = fs.statSync(file)
|
|
30
|
+
return `${stat.size}:${Math.round(stat.mtimeMs)}`
|
|
31
|
+
} catch {
|
|
32
|
+
return null
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve))
|
|
37
|
+
|
|
38
|
+
// Compares environment `a` with `b` for every page and viewport of the board.
|
|
39
|
+
// Returns { a, b, pages: { [pageId]: { [viewport]: { status, percent?, diff? } } }, changedPages }
|
|
40
|
+
// status: 'ok' (diff is the mask file name), 'missing' (one side has no capture).
|
|
41
|
+
export async function compareEnvironments(config, board, a, b, { onlyPages = null } = {}) {
|
|
42
|
+
const dir = diffsDir(config, board.id, a, b)
|
|
43
|
+
const indexFile = path.join(dir, 'index.json')
|
|
44
|
+
const dirA = capturesDir(config, board.id, a)
|
|
45
|
+
const dirB = capturesDir(config, board.id, b)
|
|
46
|
+
return withFileLock(
|
|
47
|
+
indexFile,
|
|
48
|
+
async () => {
|
|
49
|
+
const cached = readJson(indexFile)
|
|
50
|
+
const entries = cached?.format === CACHE_FORMAT && cached.entries && typeof cached.entries === 'object' ? cached.entries : {}
|
|
51
|
+
const pages = {}
|
|
52
|
+
let dirty = false
|
|
53
|
+
for (const page of board.pages) {
|
|
54
|
+
if (onlyPages && !onlyPages.includes(page.id)) continue
|
|
55
|
+
pages[page.id] = {}
|
|
56
|
+
for (const viewport of page.viewports) {
|
|
57
|
+
const name = `${page.id}.${viewport}.png`
|
|
58
|
+
const fileA = path.join(dirA, name)
|
|
59
|
+
const fileB = path.join(dirB, name)
|
|
60
|
+
const key = `${page.id}.${viewport}`
|
|
61
|
+
const sa = stamp(fileA)
|
|
62
|
+
const sb = stamp(fileB)
|
|
63
|
+
if (!sa || !sb) {
|
|
64
|
+
pages[page.id][viewport] = { status: 'missing' }
|
|
65
|
+
continue
|
|
66
|
+
}
|
|
67
|
+
const hit = entries[key]
|
|
68
|
+
if (hit && hit.a === sa && hit.b === sb && fs.existsSync(path.join(dir, name))) {
|
|
69
|
+
pages[page.id][viewport] = { status: 'ok', percent: hit.percent, diff: name }
|
|
70
|
+
continue
|
|
71
|
+
}
|
|
72
|
+
let result
|
|
73
|
+
try {
|
|
74
|
+
result = diffPngs(fs.readFileSync(fileA), fs.readFileSync(fileB))
|
|
75
|
+
} catch {
|
|
76
|
+
pages[page.id][viewport] = { status: 'missing' }
|
|
77
|
+
continue
|
|
78
|
+
}
|
|
79
|
+
writeFileAtomic(path.join(dir, name), result.png)
|
|
80
|
+
entries[key] = { a: sa, b: sb, percent: result.percent, changed: result.changed, total: result.total }
|
|
81
|
+
dirty = true
|
|
82
|
+
pages[page.id][viewport] = { status: 'ok', percent: result.percent, diff: name }
|
|
83
|
+
// Big boards: let the server answer other requests between pages.
|
|
84
|
+
await yieldToLoop()
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (dirty) writeJsonAtomic(indexFile, { format: CACHE_FORMAT, entries })
|
|
88
|
+
const changedPages = Object.values(pages).filter((viewports) =>
|
|
89
|
+
Object.values(viewports).some((item) => item.status === 'ok' && item.percent > 0),
|
|
90
|
+
).length
|
|
91
|
+
return { a, b, pages, changedPages }
|
|
92
|
+
},
|
|
93
|
+
{ timeoutMs: 120000, staleMs: 300000 },
|
|
94
|
+
)
|
|
95
|
+
}
|
package/src/discover.js
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
// Page discovery for `shapewall init --base-url`: finds a site's real pages
|
|
2
|
+
// by following the links on its home page (two levels deep at most), so a
|
|
3
|
+
// new config starts with pages that exist. Plain GETs only: no cookies, no
|
|
4
|
+
// JavaScript, no other origin (also after redirects), HTML responses only,
|
|
5
|
+
// with a page cap, a per-request timeout, a total time budget and a body cap.
|
|
6
|
+
// The base URL's own origin may be localhost or a private address: this is
|
|
7
|
+
// meant for the user's own dev server.
|
|
8
|
+
|
|
9
|
+
import { readBounded } from './sitemap.js'
|
|
10
|
+
|
|
11
|
+
export const DISCOVER_DEFAULTS = Object.freeze({
|
|
12
|
+
maxPages: 25,
|
|
13
|
+
maxDepth: 2,
|
|
14
|
+
totalMs: 10000,
|
|
15
|
+
requestMs: 4000,
|
|
16
|
+
bytesMax: 2 * 1024 * 1024,
|
|
17
|
+
concurrency: 4,
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
const REDIRECTS_MAX = 5
|
|
21
|
+
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])
|
|
22
|
+
// Links to files, not pages.
|
|
23
|
+
const FILE_EXT =
|
|
24
|
+
/\.(pdf|zip|gz|tgz|tar|rar|7z|dmg|exe|msi|apk|png|jpe?g|gif|webp|avif|svg|ico|bmp|tiff?|css|js|mjs|map|xml|json|txt|csv|rss|atom|mp3|mp4|m4a|webm|mov|wav|ogg|woff2?|ttf|otf|eot|docx?|xlsx?|pptx?)$/i
|
|
25
|
+
|
|
26
|
+
const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', ndash: '-', mdash: '-', middot: '·', hellip: '...' }
|
|
27
|
+
|
|
28
|
+
function decodeHtml(value) {
|
|
29
|
+
return String(value).replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, name) => {
|
|
30
|
+
if (name[0] === '#') {
|
|
31
|
+
const code = name[1].toLowerCase() === 'x' ? parseInt(name.slice(2), 16) : parseInt(name.slice(1), 10)
|
|
32
|
+
return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : ''
|
|
33
|
+
}
|
|
34
|
+
return ENTITIES[name.toLowerCase()] ?? match
|
|
35
|
+
})
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const clean = (value) => decodeHtml(value).replace(/\s+/g, ' ').trim()
|
|
39
|
+
|
|
40
|
+
// Comments, scripts, styles and templates hold text that looks like markup.
|
|
41
|
+
const stripNoise = (html) =>
|
|
42
|
+
String(html)
|
|
43
|
+
.replace(/<!--[\s\S]*?-->/g, '')
|
|
44
|
+
.replace(/<(script|style|template|noscript)\b[\s\S]*?<\/\1\s*>/gi, '')
|
|
45
|
+
|
|
46
|
+
const attr = (tag, name) => {
|
|
47
|
+
const match = new RegExp(`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'>]+))`, 'i').exec(tag)
|
|
48
|
+
return match ? decodeHtml(match[1] ?? match[2] ?? match[3] ?? '').trim() : null
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// The page's <title>, whitespace collapsed, or null.
|
|
52
|
+
export function htmlTitle(html) {
|
|
53
|
+
const match = /<title\b[^>]*>([\s\S]*?)<\/title\s*>/i.exec(String(html).replace(/<!--[\s\S]*?-->/g, ''))
|
|
54
|
+
const title = match ? clean(match[1]) : ''
|
|
55
|
+
return title || null
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// The href of every <a> in the page, resolved against the page URL (or its
|
|
59
|
+
// <base href>), as absolute URLs. Unparseable hrefs are dropped.
|
|
60
|
+
export function htmlLinks(html, pageUrl) {
|
|
61
|
+
const source = stripNoise(html)
|
|
62
|
+
let base = pageUrl
|
|
63
|
+
const baseTag = /<base\b[^>]*>/i.exec(source)
|
|
64
|
+
const baseHref = baseTag && attr(baseTag[0], 'href')
|
|
65
|
+
if (baseHref) {
|
|
66
|
+
try {
|
|
67
|
+
base = new URL(baseHref, pageUrl).toString()
|
|
68
|
+
} catch {
|
|
69
|
+
// keep the page URL
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
const links = []
|
|
73
|
+
for (const match of source.matchAll(/<a\b[^>]*>/gi)) {
|
|
74
|
+
const href = attr(match[0], 'href')
|
|
75
|
+
if (!href) continue
|
|
76
|
+
try {
|
|
77
|
+
links.push(new URL(href, base))
|
|
78
|
+
} catch {
|
|
79
|
+
// not a URL
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return links
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// The page a link points at on this site, as a URL string without query
|
|
86
|
+
// and fragment, or null when it leaves the site (another origin, outside
|
|
87
|
+
// the base path), is not http(s), or names a file.
|
|
88
|
+
export function pageLink(url, { origin, basePath = '' }) {
|
|
89
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') return null
|
|
90
|
+
if (url.origin !== origin) return null
|
|
91
|
+
if (basePath && url.pathname !== basePath && !url.pathname.startsWith(`${basePath}/`)) return null
|
|
92
|
+
if (FILE_EXT.test(url.pathname)) return null
|
|
93
|
+
const next = new URL(url)
|
|
94
|
+
next.search = ''
|
|
95
|
+
next.hash = ''
|
|
96
|
+
return next.toString()
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// "/about" and "/about/" are one page.
|
|
100
|
+
const pageKey = (value) => {
|
|
101
|
+
const url = new URL(value)
|
|
102
|
+
return `${url.origin}${url.pathname.replace(/\/+$/, '') || '/'}`
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// A failed request in a few words ("connection refused (is the site running?)").
|
|
106
|
+
export function describeFetchError(error) {
|
|
107
|
+
const code = error?.cause?.code || error?.code
|
|
108
|
+
if (error?.name === 'TimeoutError' || error?.name === 'AbortError' || code === 'UND_ERR_HEADERS_TIMEOUT') return 'timed out'
|
|
109
|
+
if (code === 'ECONNREFUSED') return 'connection refused (is the site running?)'
|
|
110
|
+
if (code === 'ENOTFOUND' || code === 'EAI_AGAIN') return 'host not found'
|
|
111
|
+
if (code === 'ECONNRESET') return 'connection reset'
|
|
112
|
+
return error?.cause?.message || error?.message || String(error)
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// One page: GET, redirects followed by hand within the origin, 2xx HTML only.
|
|
116
|
+
// Returns { url, html } or { error }.
|
|
117
|
+
async function fetchPage(start, { fetch, origin, signal, requestMs, bytesMax }) {
|
|
118
|
+
let url = start
|
|
119
|
+
try {
|
|
120
|
+
for (let hops = 0; ; hops += 1) {
|
|
121
|
+
const response = await fetch(url, {
|
|
122
|
+
method: 'GET',
|
|
123
|
+
headers: { accept: 'text/html, application/xhtml+xml;q=0.9, */*;q=0.1', 'user-agent': 'Shapewall init' },
|
|
124
|
+
redirect: 'manual',
|
|
125
|
+
credentials: 'omit',
|
|
126
|
+
signal: AbortSignal.any([signal, AbortSignal.timeout(requestMs)]),
|
|
127
|
+
})
|
|
128
|
+
if (REDIRECT_STATUSES.has(response.status)) {
|
|
129
|
+
await response.body?.cancel?.().catch(() => {})
|
|
130
|
+
const location = response.headers?.get?.('location')
|
|
131
|
+
if (!location) return { error: `HTTP ${response.status} without a Location` }
|
|
132
|
+
let next
|
|
133
|
+
try {
|
|
134
|
+
next = new URL(location, url)
|
|
135
|
+
} catch {
|
|
136
|
+
return { error: 'redirected to an invalid URL' }
|
|
137
|
+
}
|
|
138
|
+
if (next.origin !== origin) return { error: `redirected to another site (${next.origin})` }
|
|
139
|
+
if (hops >= REDIRECTS_MAX) return { error: `more than ${REDIRECTS_MAX} redirects` }
|
|
140
|
+
url = next.toString()
|
|
141
|
+
continue
|
|
142
|
+
}
|
|
143
|
+
if (response.url && new URL(response.url).origin !== origin) return { error: 'redirected to another site' }
|
|
144
|
+
if (!response.ok) {
|
|
145
|
+
await response.body?.cancel?.().catch(() => {})
|
|
146
|
+
return { error: `HTTP ${response.status}` }
|
|
147
|
+
}
|
|
148
|
+
const type = String(response.headers?.get?.('content-type') || '').toLowerCase()
|
|
149
|
+
if (type && !/text\/html|application\/xhtml\+xml/.test(type)) {
|
|
150
|
+
await response.body?.cancel?.().catch(() => {})
|
|
151
|
+
return { error: `not a web page (${type.split(';')[0]})` }
|
|
152
|
+
}
|
|
153
|
+
const html = (await readBounded(response, url, bytesMax)).toString('utf8')
|
|
154
|
+
if (!type && !/<(!doctype html|html|head|body|title)\b/i.test(html)) return { error: 'not a web page' }
|
|
155
|
+
return { url, html }
|
|
156
|
+
}
|
|
157
|
+
} catch (error) {
|
|
158
|
+
return { error: describeFetchError(error) }
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// Runs `task` over `items` with at most `limit` at a time, results in order.
|
|
163
|
+
async function pool(items, limit, task) {
|
|
164
|
+
const results = new Array(items.length)
|
|
165
|
+
let next = 0
|
|
166
|
+
const worker = async () => {
|
|
167
|
+
while (next < items.length) {
|
|
168
|
+
const index = next
|
|
169
|
+
next += 1
|
|
170
|
+
results[index] = await task(items[index], index)
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker))
|
|
174
|
+
return results
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// Crawls from the base URL. Returns { pages: [{ url, title }], homeTitle,
|
|
178
|
+
// home: { ok, error }, failed: [{ url, reason }], stopped } where pages are
|
|
179
|
+
// the reachable HTML pages in discovery order (home first), `failed` lists
|
|
180
|
+
// links that did not give a page, and `stopped` says why discovery ended
|
|
181
|
+
// early ('pages' at the page cap, 'time' at the time budget), or null.
|
|
182
|
+
export async function discoverPages(baseUrl, { fetch = globalThis.fetch, ...options } = {}) {
|
|
183
|
+
const settings = { ...DISCOVER_DEFAULTS, ...options }
|
|
184
|
+
const base = new URL(baseUrl)
|
|
185
|
+
base.search = ''
|
|
186
|
+
base.hash = ''
|
|
187
|
+
const origin = base.origin
|
|
188
|
+
const basePath = base.pathname.replace(/\/+$/, '')
|
|
189
|
+
const signal = AbortSignal.timeout(settings.totalMs)
|
|
190
|
+
const seen = new Set([pageKey(base.toString())])
|
|
191
|
+
const pages = []
|
|
192
|
+
const failed = []
|
|
193
|
+
let fetches = 0
|
|
194
|
+
let stopped = null
|
|
195
|
+
let homeTitle = null
|
|
196
|
+
let home = { ok: false, error: null }
|
|
197
|
+
// Each level holds links in the order they were found, so results do not
|
|
198
|
+
// depend on which response came back first.
|
|
199
|
+
let level = [base.toString()]
|
|
200
|
+
for (let depth = 0; depth <= settings.maxDepth && level.length; depth += 1) {
|
|
201
|
+
// Pages found so far, counting this level's as they come in.
|
|
202
|
+
let found = pages.length
|
|
203
|
+
const results = await pool(level, settings.concurrency, async (url) => {
|
|
204
|
+
if (found >= settings.maxPages || fetches >= settings.maxPages * 2) {
|
|
205
|
+
stopped ||= 'pages'
|
|
206
|
+
return null
|
|
207
|
+
}
|
|
208
|
+
if (signal.aborted) {
|
|
209
|
+
stopped ||= 'time'
|
|
210
|
+
return null
|
|
211
|
+
}
|
|
212
|
+
fetches += 1
|
|
213
|
+
const result = await fetchPage(url, { fetch, origin, signal, ...settings })
|
|
214
|
+
if (signal.aborted && result.error) stopped ||= 'time'
|
|
215
|
+
if (!result.error) found += 1
|
|
216
|
+
return result
|
|
217
|
+
})
|
|
218
|
+
const nextLevel = []
|
|
219
|
+
results.forEach((result, index) => {
|
|
220
|
+
if (!result) return
|
|
221
|
+
const url = level[index]
|
|
222
|
+
if (result.error) {
|
|
223
|
+
if (depth === 0) home = { ok: false, error: result.error }
|
|
224
|
+
else failed.push({ url, reason: result.error })
|
|
225
|
+
return
|
|
226
|
+
}
|
|
227
|
+
if (pages.length >= settings.maxPages) {
|
|
228
|
+
stopped ||= 'pages'
|
|
229
|
+
return
|
|
230
|
+
}
|
|
231
|
+
const title = htmlTitle(result.html)
|
|
232
|
+
if (depth === 0) {
|
|
233
|
+
home = { ok: true, error: null }
|
|
234
|
+
homeTitle = title
|
|
235
|
+
pages.push({ url, title })
|
|
236
|
+
} else {
|
|
237
|
+
// A link that redirected lands on the page it redirected to.
|
|
238
|
+
const finalKey = pageKey(result.url)
|
|
239
|
+
if (result.url !== url && seen.has(finalKey) && finalKey !== pageKey(url)) return
|
|
240
|
+
seen.add(finalKey)
|
|
241
|
+
pages.push({ url: pageLink(new URL(result.url), { origin, basePath }) || url, title })
|
|
242
|
+
}
|
|
243
|
+
if (depth >= settings.maxDepth) return
|
|
244
|
+
for (const link of htmlLinks(result.html, result.url)) {
|
|
245
|
+
const page = pageLink(link, { origin, basePath })
|
|
246
|
+
if (!page) continue
|
|
247
|
+
const key = pageKey(page)
|
|
248
|
+
if (seen.has(key)) continue
|
|
249
|
+
seen.add(key)
|
|
250
|
+
nextLevel.push(page)
|
|
251
|
+
}
|
|
252
|
+
})
|
|
253
|
+
if (stopped === 'time') break
|
|
254
|
+
level = nextLevel
|
|
255
|
+
}
|
|
256
|
+
return { pages, homeTitle, home, failed, stopped }
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// A title cut at a word boundary near `max` characters.
|
|
260
|
+
export function cutTitle(value, max = 60) {
|
|
261
|
+
const text = String(value || '').trim()
|
|
262
|
+
if (text.length <= max) return text
|
|
263
|
+
const cut = text.slice(0, max + 1)
|
|
264
|
+
const space = cut.lastIndexOf(' ')
|
|
265
|
+
return `${(space > max / 2 ? cut.slice(0, space) : text.slice(0, max)).replace(/[\s|:,;.-]+$/, '')}...`
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// Between a page's name and the site's in a <title>: bar, hyphen, en and em dash (as code points), dot, colon.
|
|
269
|
+
const TITLE_SEPARATORS = [' | ', ' - ', ` ${String.fromCodePoint(0x2013)} `, ` ${String.fromCodePoint(0x2014)} `, ' · ', ' :: ', ': ']
|
|
270
|
+
|
|
271
|
+
// A page's own title without the site name around it ("Pricing | Acme" is
|
|
272
|
+
// "Pricing" on the Acme board), or null when nothing of its own is left.
|
|
273
|
+
export function pageTitle(title, siteTitle) {
|
|
274
|
+
let text = String(title || '').trim()
|
|
275
|
+
if (!text) return null
|
|
276
|
+
if (siteTitle && text !== siteTitle) {
|
|
277
|
+
for (const sep of TITLE_SEPARATORS) {
|
|
278
|
+
if (text.endsWith(`${sep}${siteTitle}`)) text = text.slice(0, -(sep.length + siteTitle.length)).trim()
|
|
279
|
+
else if (text.startsWith(`${siteTitle}${sep}`)) text = text.slice(siteTitle.length + sep.length).trim()
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
if (!text || text === siteTitle) return null
|
|
283
|
+
return cutTitle(text, 120)
|
|
284
|
+
}
|