shapewall 0.0.0-stage → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +139 -0
- package/Dockerfile.agent +45 -0
- package/LICENSE +202 -0
- package/NOTICE +12 -0
- package/README.md +651 -2
- package/THIRD_PARTY.md +165 -0
- package/bin/shapewall.js +29 -0
- package/dist/agent/package-lock.json +8384 -0
- package/dist/client/assets/BoardChat-D5SFd8HE.js +28 -0
- package/dist/client/assets/BuzzDialog-HWXBy52W.js +1 -0
- package/dist/client/assets/ChecksPanel-DTdImWoD.js +1 -0
- package/dist/client/assets/History-CEv6b-av.js +1 -0
- package/dist/client/assets/LocalSettings-C4vriEVQ.js +1 -0
- package/dist/client/assets/Management-DCfhcEUY.js +1 -0
- package/dist/client/assets/QuickConnect-DQGgUsl1.js +1 -0
- package/dist/client/assets/ShareDialog-C_plIto0.js +2 -0
- package/dist/client/assets/app-update-dialog-2hPz_N9G.js +1 -0
- package/dist/client/assets/circle-x-BZMPi8Zf.js +1 -0
- package/dist/client/assets/index-ByQRKVit.css +2 -0
- package/dist/client/assets/index-DweQ8Qtu.js +29 -0
- package/dist/client/assets/refresh-cw-CRow0Nbq.js +1 -0
- package/dist/client/assets/runtime-Bft6x_6E.js +4 -0
- package/dist/client/assets/square-A8EnIYLR.js +1 -0
- package/dist/client/assets/tts-dist-DYlBQmwG.js +18 -0
- package/dist/client/assets/tts-en-g2p-BPUtBFGG.js +1 -0
- package/dist/client/assets/tts-ort-wasm-simd-threaded-qh98TCA_.wasm +0 -0
- package/dist/client/assets/tts-ort.wasm.bundle.min-rEkcs_ow.js +12 -0
- package/dist/client/assets/tts-rolldown-runtime-D8NudN84.js +1 -0
- package/dist/client/assets/tts-spacy-en-tokenizer-6SNMl--A.json +1 -0
- package/dist/client/assets/tts-tts.worker-Bzm6aWbv.js +1 -0
- package/dist/client/assets/tts-us_gold-DcjeNMH1.json +92589 -0
- package/dist/client/assets/tts-us_silver-C964mEB5.json +93363 -0
- package/dist/client/assets/ui-CLbws0rx.js +41 -0
- package/dist/client/build-info.json +1 -0
- package/dist/client/favicon.svg +7 -0
- package/dist/client/icons/apple-touch-icon.png +0 -0
- package/dist/client/icons/icon-192.png +0 -0
- package/dist/client/icons/icon-512.png +0 -0
- package/dist/client/icons/maskable-512.png +0 -0
- package/dist/client/index.html +25 -0
- package/dist/client/manifest.webmanifest +15 -0
- package/dist/client/sw.js +53 -0
- package/dist/client/theme-boot.js +27 -0
- package/dist/console/assets/LocalAccountsApp-BQCcmj3x.js +1 -0
- package/dist/console/assets/LocalConsole-Bb2FUDs7.js +1 -0
- package/dist/console/assets/app-update-dialog-DIeYepLZ.js +1 -0
- package/dist/console/assets/console-C7uJLLOn.js +51 -0
- package/dist/console/assets/console-R8IBY3Am.css +2 -0
- package/dist/console/build-info.json +1 -0
- package/dist/console/console.html +21 -0
- package/dist/console/favicon.svg +7 -0
- package/dist/console/icons/apple-touch-icon.png +0 -0
- package/dist/console/icons/icon-192.png +0 -0
- package/dist/console/icons/icon-512.png +0 -0
- package/dist/console/icons/maskable-512.png +0 -0
- package/dist/console/manifest.webmanifest +15 -0
- package/dist/console/theme-boot.js +27 -0
- package/docs/agent-hardening.md +342 -0
- package/docs/agent-recipes.md +187 -0
- package/docs/agent-workflow.md +320 -0
- package/docs/agents.md +894 -0
- package/docs/architecture.md +242 -0
- package/docs/buzz.md +140 -0
- package/docs/capture-warnings.md +57 -0
- package/docs/config.md +489 -0
- package/docs/deployment.md +289 -0
- package/docs/design-system.md +162 -0
- package/docs/github.md +107 -0
- package/docs/hosted.md +764 -0
- package/docs/images/board-overview.png +0 -0
- package/docs/images/journeys.png +0 -0
- package/docs/images/live-comment.png +0 -0
- package/docs/images/phone-comment.png +0 -0
- package/docs/images/phone-overview.png +0 -0
- package/docs/images/variant.png +0 -0
- package/docs/interaction-audits.md +40 -0
- package/docs/journeys.md +137 -0
- package/docs/local-accounts.md +80 -0
- package/docs/notifications.md +237 -0
- package/docs/page-checks.md +69 -0
- package/docs/performance.md +174 -0
- package/docs/starter-agent.md +325 -0
- package/docs/updates.md +87 -0
- package/examples/audits/interaction.audit.json +17 -0
- package/package.json +132 -4
- package/src/accounts/avatar.js +121 -0
- package/src/accounts/local.js +464 -0
- package/src/agent/acp/command.js +204 -0
- package/src/agent/acp/detect.js +75 -0
- package/src/agent/acp/sandbox.js +160 -0
- package/src/agent/agent-health.js +75 -0
- package/src/agent/backends/acp.js +1297 -0
- package/src/agent/backends/index.js +61 -0
- package/src/agent/backends/openai.js +308 -0
- package/src/agent/composer.js +221 -0
- package/src/agent/config.js +338 -0
- package/src/agent/context.js +191 -0
- package/src/agent/control.js +122 -0
- package/src/agent/errors.js +17 -0
- package/src/agent/feedback-loop.js +137 -0
- package/src/agent/manage.js +94 -0
- package/src/agent/next-steps.js +51 -0
- package/src/agent/owner-mcp.js +38 -0
- package/src/agent/plain-text.js +52 -0
- package/src/agent/ratings-core.js +213 -0
- package/src/agent/ratings.js +100 -0
- package/src/agent/reply-text.js +67 -0
- package/src/agent/routes.js +1065 -0
- package/src/agent/run.js +326 -0
- package/src/agent/settings-routes.js +704 -0
- package/src/agent/showable.js +23 -0
- package/src/agent/starter/approvals.js +181 -0
- package/src/agent/starter/build-image.js +90 -0
- package/src/agent/starter/cli.js +163 -0
- package/src/agent/starter/disk.js +53 -0
- package/src/agent/starter/doctor.js +180 -0
- package/src/agent/starter/egress-proxy.js +351 -0
- package/src/agent/starter/gate.js +90 -0
- package/src/agent/starter/loop.js +114 -0
- package/src/agent/starter/model-relay.js +289 -0
- package/src/agent/starter/pause.js +14 -0
- package/src/agent/starter/probe.js +272 -0
- package/src/agent/starter/prompt.js +41 -0
- package/src/agent/starter/push-guard.js +146 -0
- package/src/agent/starter/rpc.js +164 -0
- package/src/agent/starter/runtime.js +287 -0
- package/src/agent/starter/sandbox-main.js +121 -0
- package/src/agent/starter/settings.js +290 -0
- package/src/agent/starter/source.js +69 -0
- package/src/agent/starter/supervisor.js +755 -0
- package/src/agent/starter/sweep.js +46 -0
- package/src/agent/starter/tools.js +524 -0
- package/src/agent/starter/watch.js +84 -0
- package/src/agent/starter/worktree.js +409 -0
- package/src/agent/text-diff.js +210 -0
- package/src/agent/thread-history.js +223 -0
- package/src/agent/threads.js +399 -0
- package/src/agent/tools.js +200 -0
- package/src/agent/turn-changes.js +424 -0
- package/src/agent/undo-state.js +26 -0
- package/src/agent/user-settings.js +308 -0
- package/src/auth/record.js +375 -0
- package/src/broken-links.js +164 -0
- package/src/buzz/client.js +75 -0
- package/src/buzz/config.js +151 -0
- package/src/buzz/environment.js +13 -0
- package/src/buzz/gateway.js +192 -0
- package/src/buzz/routes.js +61 -0
- package/src/buzz/signer-child.js +205 -0
- package/src/buzz/store.js +89 -0
- package/src/buzz/transport.js +60 -0
- package/src/buzz/verify.js +140 -0
- package/src/capture/actions.js +159 -0
- package/src/capture/capture.js +776 -0
- package/src/capture/dom-names.js +189 -0
- package/src/capture/element-map.js +509 -0
- package/src/capture/live.js +827 -0
- package/src/capture/media-scheme.js +317 -0
- package/src/capture/network-summary.js +86 -0
- package/src/capture/pacing.js +34 -0
- package/src/capture/requests.js +167 -0
- package/src/capture/sanitize.js +752 -0
- package/src/checks/accessibility.js +134 -0
- package/src/checks/budgets.js +58 -0
- package/src/checks/cli.js +55 -0
- package/src/checks/comparison.js +113 -0
- package/src/checks/environments.js +32 -0
- package/src/checks/flow-config.js +82 -0
- package/src/checks/flow-origin.js +53 -0
- package/src/checks/flows.js +398 -0
- package/src/checks/lighthouse-auth.js +145 -0
- package/src/checks/lighthouse-findings.js +130 -0
- package/src/checks/lighthouse.js +395 -0
- package/src/checks/robots.js +85 -0
- package/src/checks/routes.js +52 -0
- package/src/checks/run-control.js +259 -0
- package/src/checks/runner.js +724 -0
- package/src/checks/schema.js +133 -0
- package/src/checks/seo-analysis.js +443 -0
- package/src/checks/seo-probe.js +190 -0
- package/src/checks/seo-snapshot.js +99 -0
- package/src/checks/store.js +214 -0
- package/src/checks/transport.js +47 -0
- package/src/cli/args.js +319 -0
- package/src/cli/capture.js +79 -0
- package/src/cli/commands.js +92 -0
- package/src/cli/comments.js +84 -0
- package/src/cli/help.js +243 -0
- package/src/cli/init.js +214 -0
- package/src/cli/misc.js +64 -0
- package/src/cli/notify.js +87 -0
- package/src/cli/open.js +22 -0
- package/src/cli/serve.js +274 -0
- package/src/cli/variant.js +82 -0
- package/src/cli/watch.js +70 -0
- package/src/cli.js +130 -0
- package/src/comment-fields.js +76 -0
- package/src/comments-cli.js +208 -0
- package/src/config-reload.js +100 -0
- package/src/config.js +1061 -0
- package/src/devices-cli.js +21 -0
- package/src/diff.js +95 -0
- package/src/discover.js +284 -0
- package/src/doctor.js +295 -0
- package/src/event-log.js +268 -0
- package/src/events-cli.js +102 -0
- package/src/events.js +113 -0
- package/src/export.js +119 -0
- package/src/feedback.js +246 -0
- package/src/github.js +551 -0
- package/src/hosted/LICENSE.md +105 -0
- package/src/hosted/access-cli.js +78 -0
- package/src/hosted/admin.js +682 -0
- package/src/hosted/agent-approvals.js +163 -0
- package/src/hosted/agent-policy.js +189 -0
- package/src/hosted/app.js +510 -0
- package/src/hosted/auth-routes.js +540 -0
- package/src/hosted/auth.js +536 -0
- package/src/hosted/backup.js +483 -0
- package/src/hosted/board-context.js +35 -0
- package/src/hosted/board-routes.js +486 -0
- package/src/hosted/board-view.js +94 -0
- package/src/hosted/clean-worker.js +24 -0
- package/src/hosted/cleaner.js +89 -0
- package/src/hosted/comments.js +355 -0
- package/src/hosted/console-routes.js +401 -0
- package/src/hosted/cookies.js +18 -0
- package/src/hosted/crypto.js +80 -0
- package/src/hosted/db.js +496 -0
- package/src/hosted/device-routes.js +175 -0
- package/src/hosted/device.js +233 -0
- package/src/hosted/events.js +63 -0
- package/src/hosted/guards.js +172 -0
- package/src/hosted/guest-view.js +65 -0
- package/src/hosted/http.js +39 -0
- package/src/hosted/import.js +300 -0
- package/src/hosted/mail.js +73 -0
- package/src/hosted/mcp-http.js +71 -0
- package/src/hosted/membership.js +114 -0
- package/src/hosted/operator.js +478 -0
- package/src/hosted/ops-cli.js +345 -0
- package/src/hosted/pages.js +146 -0
- package/src/hosted/permissions.js +113 -0
- package/src/hosted/presence.js +372 -0
- package/src/hosted/projects.js +124 -0
- package/src/hosted/public.js +109 -0
- package/src/hosted/push.js +622 -0
- package/src/hosted/quotas.js +115 -0
- package/src/hosted/ratelimit.js +96 -0
- package/src/hosted/rebind.js +316 -0
- package/src/hosted/retention.js +128 -0
- package/src/hosted/server.js +260 -0
- package/src/hosted/setup-link.js +64 -0
- package/src/hosted/setup.js +323 -0
- package/src/hosted/shares.js +444 -0
- package/src/hosted/tailscale.js +204 -0
- package/src/hosted/tenancy.js +301 -0
- package/src/hosted/token-api.js +455 -0
- package/src/hosted/token-events.js +60 -0
- package/src/hosted/tokens.js +57 -0
- package/src/hosted/turnstile.js +49 -0
- package/src/hosted/usercontent.js +154 -0
- package/src/hosted/zip.js +155 -0
- package/src/invocation.js +58 -0
- package/src/journey-paths.js +282 -0
- package/src/journeys.js +189 -0
- package/src/links.js +106 -0
- package/src/local-projects.js +182 -0
- package/src/lock.js +193 -0
- package/src/mcp.js +417 -0
- package/src/mentions.js +151 -0
- package/src/net.js +15 -0
- package/src/notify/config.js +275 -0
- package/src/notify/index.js +301 -0
- package/src/notify/lease.js +73 -0
- package/src/notify/messages.js +212 -0
- package/src/notify/run.js +174 -0
- package/src/pixeldiff.js +49 -0
- package/src/png.js +143 -0
- package/src/reactions.js +111 -0
- package/src/remote/cli.js +133 -0
- package/src/remote/credentials.js +62 -0
- package/src/remote/ids.js +12 -0
- package/src/remote/login.js +155 -0
- package/src/remote/remote-mcp.js +237 -0
- package/src/remote/remote-watch.js +291 -0
- package/src/remote/remote.js +313 -0
- package/src/remote/share-state.js +78 -0
- package/src/remote/share.js +369 -0
- package/src/remote/terminal.js +13 -0
- package/src/serve-state.js +64 -0
- package/src/server/app.js +824 -0
- package/src/server/devices.js +186 -0
- package/src/server/local-console.js +28 -0
- package/src/server/local-user.js +59 -0
- package/src/server/pairing-routes.js +159 -0
- package/src/server/share-routes.js +89 -0
- package/src/server/tailscale.js +97 -0
- package/src/sitemap.js +431 -0
- package/src/store.js +983 -0
- package/src/tokens.js +11 -0
- package/src/update/routes.js +245 -0
- package/src/watch.js +403 -0
- package/src/which.js +58 -0
|
@@ -0,0 +1,622 @@
|
|
|
1
|
+
// `shapewall push`, server side. A push uploads one board: its definition
|
|
2
|
+
// (pages, groups, journeys; never auth settings, actions or paths), then each
|
|
3
|
+
// file, then a commit with the manifest and variants.
|
|
4
|
+
//
|
|
5
|
+
// Uploads are untrusted, even from our own CLI:
|
|
6
|
+
// - file names must match the names the local tool writes, for pages and
|
|
7
|
+
// viewports on the pushed board;
|
|
8
|
+
// - live HTML goes through the same sanitizer as a local capture
|
|
9
|
+
// (sanitizeDocument, then the CSP meta tag) and the same 5 MB cap;
|
|
10
|
+
// - element maps (<page>.<viewport>.elements.json) are parsed and rebuilt
|
|
11
|
+
// field by field (cleanElementMap), capped at 3 MB;
|
|
12
|
+
// - PNGs are decoded and re-encoded (pngjs), which drops every chunk but the
|
|
13
|
+
// pixels, after a header check on their size;
|
|
14
|
+
// - the manifest and variants are rebuilt field by field.
|
|
15
|
+
// Files land in a staging folder; the commit swaps it in whole, so a board
|
|
16
|
+
// never shows half a push.
|
|
17
|
+
|
|
18
|
+
import fs from 'node:fs'
|
|
19
|
+
import path from 'node:path'
|
|
20
|
+
import { PNG } from 'pngjs'
|
|
21
|
+
import { ID_RE, frameSize, normalizeConfig } from '../config.js'
|
|
22
|
+
import { FILE_MAX as HTML_MAX, withCspMeta } from '../capture/live.js'
|
|
23
|
+
import { CssTooLargeError, sanitizeDocument } from '../capture/sanitize.js'
|
|
24
|
+
import { ELEMENT_MAP_LIMITS, ElementMapError, parseElementMap } from '../capture/element-map.js'
|
|
25
|
+
import { CAPTURE_FILE_RE, VERSION_FILE_RE, VERSION_RE } from '../store.js'
|
|
26
|
+
import { ID_PATTERNS, newId } from './crypto.js'
|
|
27
|
+
import { transaction } from './db.js'
|
|
28
|
+
import { openPushIds, staleOpenPushes, tenant } from './tenancy.js'
|
|
29
|
+
import { mergeCheckEnvironments, validateCheckEnvironments } from '../checks/environments.js'
|
|
30
|
+
import { projectDir } from './projects.js'
|
|
31
|
+
import { CHECKS_BUNDLE_MAX, validateChecksBundle } from '../checks/transport.js'
|
|
32
|
+
import { listReports } from '../checks/store.js'
|
|
33
|
+
import { REPORTS_MAX, validateReport } from '../checks/schema.js'
|
|
34
|
+
import { cleanNetworkSummary } from '../capture/network-summary.js'
|
|
35
|
+
|
|
36
|
+
export const PUSH_LIMITS = {
|
|
37
|
+
checks: CHECKS_BUNDLE_MAX,
|
|
38
|
+
png: 10 * 1024 * 1024,
|
|
39
|
+
html: HTML_MAX,
|
|
40
|
+
elements: ELEMENT_MAP_LIMITS.bytes,
|
|
41
|
+
files: 5000,
|
|
42
|
+
bytes: 1024 * 1024 * 1024,
|
|
43
|
+
pages: 500,
|
|
44
|
+
variants: 200,
|
|
45
|
+
boardsPerProject: 10,
|
|
46
|
+
openMs: 60 * 60_000,
|
|
47
|
+
// All stylesheet text of one live HTML file (style elements and attributes).
|
|
48
|
+
css: 4 * 1024 * 1024,
|
|
49
|
+
// Decoded size of one PNG: 16 megapixels is 64 MB of RGBA.
|
|
50
|
+
pixels: 16_000_000,
|
|
51
|
+
// Everything a workspace stores (committed boards plus open pushes).
|
|
52
|
+
workspaceBytes: 5 * 1024 * 1024 * 1024,
|
|
53
|
+
}
|
|
54
|
+
const PNG_MAX_WIDTH = 3840
|
|
55
|
+
const PNG_MAX_HEIGHT = 8000
|
|
56
|
+
|
|
57
|
+
export class PushError extends Error {
|
|
58
|
+
constructor(status, code, message) {
|
|
59
|
+
super(message)
|
|
60
|
+
this.statusCode = status
|
|
61
|
+
this.code = code
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const isObject = (value) => Boolean(value) && typeof value === 'object' && !Array.isArray(value)
|
|
66
|
+
const str = (value, max) => (typeof value === 'string' ? value.slice(0, max) : undefined)
|
|
67
|
+
const num = (value) => (typeof value === 'number' && Number.isFinite(value) ? value : undefined)
|
|
68
|
+
|
|
69
|
+
// Only the fields the board shows. normalizeConfig then validates them the
|
|
70
|
+
// same way it validates a local config.
|
|
71
|
+
export function cleanDefinition(raw) {
|
|
72
|
+
if (!isObject(raw) || !isObject(raw.board)) throw new PushError(400, 'invalid_board', 'Send { title, viewports, board }.')
|
|
73
|
+
const board = raw.board
|
|
74
|
+
if (!Array.isArray(board.pages) || board.pages.length > PUSH_LIMITS.pages) {
|
|
75
|
+
throw new PushError(400, 'invalid_board', `A pushed board has 1 to ${PUSH_LIMITS.pages} pages.`)
|
|
76
|
+
}
|
|
77
|
+
const picked = {
|
|
78
|
+
title: raw.title,
|
|
79
|
+
viewports: raw.viewports,
|
|
80
|
+
versions: Number.isInteger(raw.versions) ? raw.versions : undefined,
|
|
81
|
+
boards: [
|
|
82
|
+
{
|
|
83
|
+
id: board.id,
|
|
84
|
+
title: board.title,
|
|
85
|
+
baseUrl: board.baseUrl,
|
|
86
|
+
publicUrl: board.publicUrl ?? undefined,
|
|
87
|
+
groups: board.groups,
|
|
88
|
+
pages: board.pages.map((page) =>
|
|
89
|
+
isObject(page)
|
|
90
|
+
? {
|
|
91
|
+
id: page.id,
|
|
92
|
+
path: page.path,
|
|
93
|
+
title: page.title,
|
|
94
|
+
group: page.group,
|
|
95
|
+
note: page.note || undefined,
|
|
96
|
+
viewports: page.viewports,
|
|
97
|
+
frameHeight: page.frameHeight ?? undefined,
|
|
98
|
+
}
|
|
99
|
+
: page,
|
|
100
|
+
),
|
|
101
|
+
journeys: board.journeys,
|
|
102
|
+
},
|
|
103
|
+
],
|
|
104
|
+
}
|
|
105
|
+
let config
|
|
106
|
+
try {
|
|
107
|
+
config = normalizeConfig(JSON.parse(JSON.stringify(picked)), { dir: '/nonexistent' })
|
|
108
|
+
} catch (error) {
|
|
109
|
+
throw new PushError(400, 'invalid_board', error.problems ? `Invalid board: ${error.problems.slice(0, 5).join('; ')}` : 'Invalid board.')
|
|
110
|
+
}
|
|
111
|
+
// Capture-only settings never travel: auth, block lists, setup, seeds.
|
|
112
|
+
// eslint-disable-next-line no-unused-vars
|
|
113
|
+
const { auth, block, reducedMotion, localStorage, setup, viewports, ...normalized } = config.boards[0]
|
|
114
|
+
return {
|
|
115
|
+
title: config.title,
|
|
116
|
+
viewports: config.viewports,
|
|
117
|
+
versions: config.versions,
|
|
118
|
+
board: {
|
|
119
|
+
...normalized,
|
|
120
|
+
// One environment hosted (compare needs more than a push for now).
|
|
121
|
+
environments: [{ ...normalized.environments[0], id: 'default', title: 'Default' }],
|
|
122
|
+
pages: normalized.pages.map((page) => ({ ...page, waitFor: null, actions: [] })),
|
|
123
|
+
},
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function pngSize(buffer) {
|
|
128
|
+
const signature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])
|
|
129
|
+
if (buffer.length < 33 || !buffer.subarray(0, 8).equals(signature) || buffer.toString('latin1', 12, 16) !== 'IHDR') return null
|
|
130
|
+
return { width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20) }
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export function cleanPng(buffer) {
|
|
134
|
+
if (buffer.length > PUSH_LIMITS.png) throw new PushError(413, 'too_large', `Images are limited to ${PUSH_LIMITS.png / 1048576} MB.`)
|
|
135
|
+
const size = pngSize(buffer)
|
|
136
|
+
if (!size) throw new PushError(400, 'invalid_png', 'Not a PNG.')
|
|
137
|
+
if (
|
|
138
|
+
!size.width ||
|
|
139
|
+
!size.height ||
|
|
140
|
+
size.width > PNG_MAX_WIDTH ||
|
|
141
|
+
size.height > PNG_MAX_HEIGHT ||
|
|
142
|
+
size.width * size.height > PUSH_LIMITS.pixels
|
|
143
|
+
) {
|
|
144
|
+
throw new PushError(
|
|
145
|
+
400,
|
|
146
|
+
'invalid_png',
|
|
147
|
+
`Images are at most ${PNG_MAX_WIDTH}x${PNG_MAX_HEIGHT} pixels and ${PUSH_LIMITS.pixels / 1e6} megapixels.`,
|
|
148
|
+
)
|
|
149
|
+
}
|
|
150
|
+
let image
|
|
151
|
+
try {
|
|
152
|
+
image = PNG.sync.read(buffer)
|
|
153
|
+
} catch {
|
|
154
|
+
throw new PushError(400, 'invalid_png', 'The PNG could not be decoded.')
|
|
155
|
+
}
|
|
156
|
+
return PNG.sync.write(image, { colorType: 6 })
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// The same pass a local capture's live HTML gets after the snapshot.
|
|
160
|
+
export function cleanHtml(buffer, board) {
|
|
161
|
+
if (buffer.length > PUSH_LIMITS.html) throw new PushError(413, 'too_large', `Live HTML is limited to ${PUSH_LIMITS.html / 1048576} MB.`)
|
|
162
|
+
let text
|
|
163
|
+
try {
|
|
164
|
+
text = new TextDecoder('utf-8', { fatal: true }).decode(buffer)
|
|
165
|
+
} catch {
|
|
166
|
+
throw new PushError(400, 'invalid_html', 'Live HTML must be UTF-8.')
|
|
167
|
+
}
|
|
168
|
+
let html
|
|
169
|
+
try {
|
|
170
|
+
html = withCspMeta(sanitizeDocument(text, { base: board.publicUrl || board.baseUrl, maxCss: PUSH_LIMITS.css }))
|
|
171
|
+
} catch (error) {
|
|
172
|
+
if (error instanceof CssTooLargeError)
|
|
173
|
+
throw new PushError(413, 'too_large', `The CSS of one live HTML file is limited to ${PUSH_LIMITS.css / 1048576} MB.`)
|
|
174
|
+
throw error
|
|
175
|
+
}
|
|
176
|
+
if (Buffer.byteLength(html) > PUSH_LIMITS.html) throw new PushError(413, 'too_large', 'Live HTML is over the cap after cleaning.')
|
|
177
|
+
return Buffer.from(html)
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// An element map: valid JSON in the map format, rebuilt, re-serialized,
|
|
181
|
+
// measured at its viewport's size (`size`, from the pushed definition).
|
|
182
|
+
export function cleanElements(buffer, size = null) {
|
|
183
|
+
if (buffer.length > PUSH_LIMITS.elements)
|
|
184
|
+
throw new PushError(413, 'too_large', `An element map is limited to ${PUSH_LIMITS.elements / 1048576} MB.`)
|
|
185
|
+
let text
|
|
186
|
+
try {
|
|
187
|
+
text = new TextDecoder('utf-8', { fatal: true }).decode(buffer)
|
|
188
|
+
} catch {
|
|
189
|
+
throw new PushError(400, 'invalid_elements', 'An element map must be UTF-8 JSON.')
|
|
190
|
+
}
|
|
191
|
+
try {
|
|
192
|
+
const map = parseElementMap(text)
|
|
193
|
+
// `size.heights`: the frame heights a map may have (see elementMapSize).
|
|
194
|
+
if (size && (map.width !== size.width || !(size.heights || [size.height]).includes(map.height)))
|
|
195
|
+
throw new ElementMapError(`measured at ${map.width}x${map.height}, not at its frame's ${size.width}x${size.height}`)
|
|
196
|
+
return Buffer.from(JSON.stringify(map))
|
|
197
|
+
} catch (error) {
|
|
198
|
+
if (error instanceof ElementMapError) throw new PushError(400, 'invalid_elements', `Element map: ${error.message}.`)
|
|
199
|
+
throw error
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const kindOf = (suffix) => (suffix === '.html' ? 'html' : suffix === '.elements.json' ? 'elements' : 'png')
|
|
204
|
+
|
|
205
|
+
// Which file a name is: { kind: png|html|elements, owner, viewport } or null.
|
|
206
|
+
// The size an element map was measured at: its frame. A page may set its own
|
|
207
|
+
// frame height; a variant takes its page's, and which page that is arrives
|
|
208
|
+
// later (variants.json), so a variant's map may have any of the board's
|
|
209
|
+
// frame heights for that viewport.
|
|
210
|
+
function elementMapSize(definition, parsed, area) {
|
|
211
|
+
const size = definition.viewports?.[parsed.viewport]
|
|
212
|
+
if (!size) return null
|
|
213
|
+
const pages = definition.board.pages
|
|
214
|
+
const owners = area === 'variants' ? pages : pages.filter((item) => item.id === parsed.owner)
|
|
215
|
+
const heights = new Set(area === 'variants' ? [size.height] : [])
|
|
216
|
+
for (const page of owners) heights.add(frameSize(size, page, parsed.viewport).height)
|
|
217
|
+
return { width: size.width, height: [...heights][0], heights: [...heights] }
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function parseName(area, name, board, variantIds = null) {
|
|
221
|
+
if (area === 'captures' || area === 'variants') {
|
|
222
|
+
const match = CAPTURE_FILE_RE.exec(name)
|
|
223
|
+
if (!match) return null
|
|
224
|
+
const owner = name.slice(0, name.indexOf('.'))
|
|
225
|
+
const viewport = match[1]
|
|
226
|
+
if (area === 'captures') {
|
|
227
|
+
const page = board.pages.find((item) => item.id === owner)
|
|
228
|
+
if (!page || !page.viewports.includes(viewport)) return null
|
|
229
|
+
} else if (!ID_RE.test(owner) || (variantIds && !variantIds.has(owner))) return null
|
|
230
|
+
return { kind: kindOf(match[2]), owner, viewport }
|
|
231
|
+
}
|
|
232
|
+
if (area === 'versions') {
|
|
233
|
+
const match = VERSION_FILE_RE.exec(name)
|
|
234
|
+
if (!match) return null
|
|
235
|
+
const page = board.pages.find((item) => item.id === match[1])
|
|
236
|
+
if (!page || !page.viewports.includes(match[2])) return null
|
|
237
|
+
return { kind: match[4] === '.html' ? 'html' : 'png', owner: match[1], viewport: match[2] }
|
|
238
|
+
}
|
|
239
|
+
return null
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function cleanManifest(raw, board, viewports, files) {
|
|
243
|
+
const manifest = {
|
|
244
|
+
board: board.id,
|
|
245
|
+
capturedAt:
|
|
246
|
+
typeof raw?.capturedAt === 'string' && Number.isFinite(Date.parse(raw.capturedAt)) ? new Date(raw.capturedAt).toISOString() : null,
|
|
247
|
+
durationMs: num(raw?.durationMs) ?? null,
|
|
248
|
+
viewports,
|
|
249
|
+
pages: {},
|
|
250
|
+
}
|
|
251
|
+
const pageIds = new Set(board.pages.map((page) => page.id))
|
|
252
|
+
for (const page of board.pages) {
|
|
253
|
+
manifest.pages[page.id] = {}
|
|
254
|
+
for (const viewport of page.viewports) {
|
|
255
|
+
const entry = raw?.pages?.[page.id]?.[viewport]
|
|
256
|
+
if (!isObject(entry)) continue
|
|
257
|
+
const ok = entry.status === 'ok' && files.has(`captures/${page.id}.${viewport}.png`)
|
|
258
|
+
const clean = { status: ok ? 'ok' : 'failed' }
|
|
259
|
+
const networkSummary = cleanNetworkSummary(entry.networkSummary)
|
|
260
|
+
if (networkSummary) clean.networkSummary = networkSummary
|
|
261
|
+
if (Number.isInteger(entry.httpStatus) && entry.httpStatus >= 100 && entry.httpStatus <= 599) clean.httpStatus = entry.httpStatus
|
|
262
|
+
if (entry.expectedStatus === true && clean.httpStatus !== undefined) clean.expectedStatus = true
|
|
263
|
+
const url = str(entry.url, 2000)
|
|
264
|
+
if (url && /^https?:\/\//.test(url)) clean.url = url
|
|
265
|
+
if (!ok) clean.error = str(entry.error, 300) || (entry.status === 'ok' ? 'not pushed' : 'failed')
|
|
266
|
+
if (num(entry.ms) !== undefined) clean.ms = num(entry.ms)
|
|
267
|
+
if (num(entry.bytes) !== undefined) clean.bytes = num(entry.bytes)
|
|
268
|
+
if (ok && typeof entry.version === 'string' && VERSION_RE.test(entry.version)) clean.version = entry.version
|
|
269
|
+
if (isObject(entry.html)) {
|
|
270
|
+
const html = { status: entry.html.status === 'ok' && files.has(`captures/${page.id}.${viewport}.html`) ? 'ok' : 'failed' }
|
|
271
|
+
for (const key of ['bytes', 'images', 'imagesDropped', 'fonts', 'fontsDropped'])
|
|
272
|
+
if (num(entry.html[key]) !== undefined) html[key] = num(entry.html[key])
|
|
273
|
+
if (typeof entry.html.sha256 === 'string' && /^[a-f0-9]{64}$/.test(entry.html.sha256)) html.sha256 = entry.html.sha256
|
|
274
|
+
if (html.status !== 'ok') html.error = str(entry.html.error, 200) || 'not pushed'
|
|
275
|
+
clean.html = html
|
|
276
|
+
}
|
|
277
|
+
if (Array.isArray(entry.links)) {
|
|
278
|
+
const links = [...new Set(entry.links.filter((id) => typeof id === 'string' && pageIds.has(id)))].slice(0, 500)
|
|
279
|
+
if (links.length) clean.links = links
|
|
280
|
+
}
|
|
281
|
+
manifest.pages[page.id][viewport] = clean
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
return manifest
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function cleanVariants(raw, board) {
|
|
288
|
+
const pages = new Set(board.pages.map((page) => page.id))
|
|
289
|
+
const text = (value, max) => (typeof value === 'string' ? value.trim().slice(0, max) : '')
|
|
290
|
+
const out = []
|
|
291
|
+
for (const item of Array.isArray(raw) ? raw.slice(0, PUSH_LIMITS.variants) : []) {
|
|
292
|
+
if (!isObject(item) || !ID_RE.test(String(item.id || '')) || !pages.has(item.pageId)) continue
|
|
293
|
+
if (out.some((existing) => existing.id === item.id)) continue
|
|
294
|
+
out.push({
|
|
295
|
+
id: item.id,
|
|
296
|
+
pageId: item.pageId,
|
|
297
|
+
title: text(item.title, 120) || item.id,
|
|
298
|
+
note: text(item.note, 1000),
|
|
299
|
+
source: text(item.source, 500),
|
|
300
|
+
createdAt: typeof item.createdAt === 'string' && Number.isFinite(Date.parse(item.createdAt)) ? item.createdAt : null,
|
|
301
|
+
})
|
|
302
|
+
}
|
|
303
|
+
return out
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
export function createPushStore(
|
|
307
|
+
t,
|
|
308
|
+
{ db, dataDir, workspaceQuota = PUSH_LIMITS.workspaceBytes, boardsPerProject = PUSH_LIMITS.boardsPerProject },
|
|
309
|
+
) {
|
|
310
|
+
const stagingDir = (pushId) => {
|
|
311
|
+
if (!ID_PATTERNS.push.test(pushId)) throw new Error('invalid push id')
|
|
312
|
+
return path.join(dataDir, 'staging', pushId)
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
const get = (pushId) =>
|
|
316
|
+
ID_PATTERNS.push.test(String(pushId || ''))
|
|
317
|
+
? t.get('SELECT * FROM pushes WHERE workspace_id = :ws AND id = :id', { id: pushId }) || null
|
|
318
|
+
: null
|
|
319
|
+
|
|
320
|
+
function sweep(now) {
|
|
321
|
+
const cutoff = new Date(now.getTime() - PUSH_LIMITS.openMs).toISOString()
|
|
322
|
+
for (const row of t.all("SELECT id FROM pushes WHERE workspace_id = :ws AND status = 'open' AND created_at < :cutoff", { cutoff }))
|
|
323
|
+
expire(row.id, now)
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
function start({ projectId, tokenId, raw, now = new Date() }) {
|
|
327
|
+
const definition = cleanDefinition(raw)
|
|
328
|
+
sweep(now)
|
|
329
|
+
const boards = t.all('SELECT board_id FROM boards WHERE workspace_id = :ws AND project_id = :project', { project: projectId })
|
|
330
|
+
if (!boards.some((row) => row.board_id === definition.board.id) && boards.length >= boardsPerProject) {
|
|
331
|
+
throw new PushError(409, 'too_many_boards', `A project holds at most ${boardsPerProject} boards.`)
|
|
332
|
+
}
|
|
333
|
+
const id = newId('ps')
|
|
334
|
+
t.run(
|
|
335
|
+
`INSERT INTO pushes (workspace_id, id, project_id, board_id, token_id, definition, status, created_at)
|
|
336
|
+
VALUES (:ws, :id, :project, :board, :token, :definition, 'open', :at)`,
|
|
337
|
+
{
|
|
338
|
+
id,
|
|
339
|
+
project: projectId,
|
|
340
|
+
board: definition.board.id,
|
|
341
|
+
token: tokenId || null,
|
|
342
|
+
definition: JSON.stringify(definition),
|
|
343
|
+
at: now.toISOString(),
|
|
344
|
+
},
|
|
345
|
+
)
|
|
346
|
+
fs.mkdirSync(stagingDir(id), { recursive: true })
|
|
347
|
+
return {
|
|
348
|
+
id,
|
|
349
|
+
board: definition.board.id,
|
|
350
|
+
limits: {
|
|
351
|
+
checks: PUSH_LIMITS.checks,
|
|
352
|
+
png: PUSH_LIMITS.png,
|
|
353
|
+
html: PUSH_LIMITS.html,
|
|
354
|
+
elements: PUSH_LIMITS.elements,
|
|
355
|
+
files: PUSH_LIMITS.files,
|
|
356
|
+
bytes: PUSH_LIMITS.bytes,
|
|
357
|
+
},
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function expire(pushId, now) {
|
|
362
|
+
fs.rmSync(stagingDir(pushId), { recursive: true, force: true })
|
|
363
|
+
t.run("UPDATE pushes SET status = 'failed', finished_at = :at WHERE workspace_id = :ws AND id = :id AND status = 'open'", {
|
|
364
|
+
id: pushId,
|
|
365
|
+
at: now.toISOString(),
|
|
366
|
+
})
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
function openPush(pushId, projectId, now) {
|
|
370
|
+
const push = get(pushId)
|
|
371
|
+
if (!push || push.project_id !== projectId) throw new PushError(404, 'unknown_push', 'No such push.')
|
|
372
|
+
if (push.status !== 'open') throw new PushError(409, 'push_closed', 'This push is already finished.')
|
|
373
|
+
if (Date.parse(push.created_at) + PUSH_LIMITS.openMs <= now.getTime()) {
|
|
374
|
+
expire(push.id, now)
|
|
375
|
+
throw new PushError(409, 'push_expired', 'This push is more than an hour old; start a new one.')
|
|
376
|
+
}
|
|
377
|
+
return { ...push, definition: JSON.parse(push.definition) }
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
// Bytes the workspace holds apart from `boardId` of `projectId` (which a
|
|
381
|
+
// push replaces): committed boards plus every open push.
|
|
382
|
+
function workspaceBytesBesides(projectId, boardId) {
|
|
383
|
+
const boards = t.get(
|
|
384
|
+
'SELECT COALESCE(SUM(bytes), 0) AS n FROM boards WHERE workspace_id = :ws AND NOT (project_id = :project AND board_id = :board)',
|
|
385
|
+
{ project: projectId, board: boardId },
|
|
386
|
+
).n
|
|
387
|
+
const open = t.get("SELECT COALESCE(SUM(bytes), 0) AS n FROM pushes WHERE workspace_id = :ws AND status = 'open'").n
|
|
388
|
+
return boards + open
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// Cleans the file off the event loop (cleaner), then writes it to the
|
|
392
|
+
// push's staging folder. The push is checked again after cleaning: it may
|
|
393
|
+
// have been committed or expired meanwhile.
|
|
394
|
+
async function receive({ pushId, projectId, area, name, body, cleaner, now = () => new Date() }) {
|
|
395
|
+
const first = openPush(pushId, projectId, now())
|
|
396
|
+
const board = first.definition.board
|
|
397
|
+
const parsed = parseName(String(area), String(name), board)
|
|
398
|
+
if (!parsed) throw new PushError(400, 'invalid_file', 'That file name is not one Shapewall writes for this board.')
|
|
399
|
+
if (!Buffer.isBuffer(body) || !body.length) throw new PushError(400, 'empty_file', 'Send the file as the request body.')
|
|
400
|
+
if (first.files >= PUSH_LIMITS.files) throw new PushError(413, 'too_many_files', `A push holds at most ${PUSH_LIMITS.files} files.`)
|
|
401
|
+
const limit = PUSH_LIMITS[parsed.kind]
|
|
402
|
+
if (body.length > limit) throw new PushError(413, 'too_large', `This file is over the ${limit / 1048576} MB cap.`)
|
|
403
|
+
const base = board.publicUrl || board.baseUrl
|
|
404
|
+
// Maps are small JSON: cleaned here. HTML and PNGs go to the worker pool.
|
|
405
|
+
const clean =
|
|
406
|
+
parsed.kind === 'elements'
|
|
407
|
+
? cleanElements(body, elementMapSize(first.definition, parsed, area))
|
|
408
|
+
: cleaner
|
|
409
|
+
? await cleaner.clean(parsed.kind, body, base)
|
|
410
|
+
: parsed.kind === 'html'
|
|
411
|
+
? cleanHtml(body, board)
|
|
412
|
+
: cleanPng(body)
|
|
413
|
+
const push = openPush(pushId, projectId, now())
|
|
414
|
+
const dir = path.join(stagingDir(push.id), area)
|
|
415
|
+
const file = path.join(dir, name)
|
|
416
|
+
if (path.dirname(file) !== dir) throw new PushError(400, 'invalid_file', 'Bad file name.')
|
|
417
|
+
const previous = fs.existsSync(file) ? fs.statSync(file).size : null
|
|
418
|
+
// Checked again on the fresh row: uploads cleaned in parallel all passed
|
|
419
|
+
// the first check before any of them was counted.
|
|
420
|
+
if (previous === null && push.files >= PUSH_LIMITS.files)
|
|
421
|
+
throw new PushError(413, 'too_many_files', `A push holds at most ${PUSH_LIMITS.files} files.`)
|
|
422
|
+
const grow = clean.length - (previous ?? 0)
|
|
423
|
+
if (push.bytes + grow > PUSH_LIMITS.bytes) throw new PushError(413, 'too_large', 'This push is over the size cap.')
|
|
424
|
+
if (workspaceBytesBesides(projectId, board.id) + grow > workspaceQuota) {
|
|
425
|
+
throw new PushError(413, 'quota_exceeded', 'This workspace is over its storage quota.')
|
|
426
|
+
}
|
|
427
|
+
fs.mkdirSync(dir, { recursive: true })
|
|
428
|
+
fs.writeFileSync(file, clean)
|
|
429
|
+
t.run('UPDATE pushes SET files = files + :files, bytes = bytes + :bytes WHERE workspace_id = :ws AND id = :id', {
|
|
430
|
+
id: push.id,
|
|
431
|
+
files: previous === null ? 1 : 0,
|
|
432
|
+
bytes: grow,
|
|
433
|
+
})
|
|
434
|
+
return { name, bytes: clean.length }
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
// Writes the cleaned manifest and variants, swaps the staging folder in
|
|
438
|
+
// and records the board definition.
|
|
439
|
+
function commit({ pushId, projectId, manifest, variants, checks, now = new Date() }) {
|
|
440
|
+
const push = openPush(pushId, projectId, now)
|
|
441
|
+
const { board, viewports } = push.definition
|
|
442
|
+
const staging = stagingDir(push.id)
|
|
443
|
+
let cleanChecks = { schemaVersion: 1, reports: [] }
|
|
444
|
+
if (checks !== undefined) {
|
|
445
|
+
try {
|
|
446
|
+
cleanChecks = validateChecksBundle(checks, board, viewports)
|
|
447
|
+
} catch {
|
|
448
|
+
throw new PushError(400, 'invalid_checks', 'Invalid or oversized checks bundle for this board.')
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
const previousRow = t.get('SELECT definition FROM boards WHERE workspace_id = :ws AND project_id = :project AND board_id = :board', {
|
|
452
|
+
project: projectId,
|
|
453
|
+
board: board.id,
|
|
454
|
+
})
|
|
455
|
+
const previousDefinition = previousRow ? JSON.parse(previousRow.definition) : null
|
|
456
|
+
const previousEnvironments = previousDefinition
|
|
457
|
+
? validateCheckEnvironments(previousDefinition.checksEnvironments || [{ id: 'default', captured: true }])
|
|
458
|
+
: []
|
|
459
|
+
let checksEnvironments
|
|
460
|
+
try {
|
|
461
|
+
checksEnvironments = mergeCheckEnvironments(
|
|
462
|
+
previousEnvironments.length || checks !== undefined ? previousEnvironments : [{ id: 'default', captured: true }],
|
|
463
|
+
checks === undefined ? null : cleanChecks.environments || [{ id: 'default', captured: true }],
|
|
464
|
+
)
|
|
465
|
+
} catch {
|
|
466
|
+
throw new PushError(
|
|
467
|
+
400,
|
|
468
|
+
'invalid_checks_environments',
|
|
469
|
+
'Diagnostic environments exceed their bound or conflict. Nothing was committed.',
|
|
470
|
+
)
|
|
471
|
+
}
|
|
472
|
+
// Existing reports are read against their retained descriptors, never just
|
|
473
|
+
// the new push's one capture environment. Ordinary pushes preserve history.
|
|
474
|
+
const existingReports = previousDefinition
|
|
475
|
+
? listReports(
|
|
476
|
+
{
|
|
477
|
+
outputDir: projectDir(dataDir, t.workspaceId, projectId),
|
|
478
|
+
boards: [board],
|
|
479
|
+
viewports,
|
|
480
|
+
checksEnvironments: previousEnvironments,
|
|
481
|
+
},
|
|
482
|
+
board.id,
|
|
483
|
+
)
|
|
484
|
+
: []
|
|
485
|
+
push.definition.checksEnvironments = checksEnvironments
|
|
486
|
+
const merged = new Map(existingReports.map((r) => [r.id, r]))
|
|
487
|
+
for (const report of cleanChecks.reports) {
|
|
488
|
+
const existing = merged.get(report.id)
|
|
489
|
+
if (existing && JSON.stringify(existing) !== JSON.stringify(report)) {
|
|
490
|
+
// Older clients remapped the source's first environment to default.
|
|
491
|
+
// Recognize that exact legacy representation on replay, retaining the
|
|
492
|
+
// immutable stored report rather than rewriting its environment.
|
|
493
|
+
const legacy =
|
|
494
|
+
report.environment !== 'default' &&
|
|
495
|
+
existing.environment === 'default' &&
|
|
496
|
+
existing.profile.sourceEnvironment === report.environment
|
|
497
|
+
? validateReport({ ...report, environment: 'default', profile: { ...report.profile, sourceEnvironment: report.environment } })
|
|
498
|
+
: null
|
|
499
|
+
if (!legacy || JSON.stringify(existing) !== JSON.stringify(legacy))
|
|
500
|
+
throw new PushError(409, 'check_conflict', 'A check report id already has different immutable content.')
|
|
501
|
+
continue
|
|
502
|
+
}
|
|
503
|
+
merged.set(report.id, report)
|
|
504
|
+
}
|
|
505
|
+
const keptReports = [...merged.values()].sort((a, b) => b.auditedAt.localeCompare(a.auditedAt)).slice(0, REPORTS_MAX)
|
|
506
|
+
const keptIds = new Set(keptReports.map((report) => report.id))
|
|
507
|
+
if (cleanChecks.reports.some((report) => !keptIds.has(report.id)))
|
|
508
|
+
throw new PushError(
|
|
509
|
+
409,
|
|
510
|
+
'checks_retention',
|
|
511
|
+
'An explicitly included check report falls outside the retained history limit. Nothing was committed.',
|
|
512
|
+
)
|
|
513
|
+
const checkBytes = keptReports.reduce((n, r) => n + Buffer.byteLength(JSON.stringify(r)), 0)
|
|
514
|
+
if (push.bytes + checkBytes > PUSH_LIMITS.bytes) throw new PushError(413, 'too_large', 'This push exceeds its storage cap.')
|
|
515
|
+
const cleanList = cleanVariants(variants, board)
|
|
516
|
+
const variantIds = new Set(cleanList.map((item) => item.id))
|
|
517
|
+
const files = new Set()
|
|
518
|
+
for (const area of ['captures', 'variants', 'versions']) {
|
|
519
|
+
const dir = path.join(staging, area)
|
|
520
|
+
if (!fs.existsSync(dir)) continue
|
|
521
|
+
for (const name of fs.readdirSync(dir)) {
|
|
522
|
+
// Files of variants that are not in the final list go.
|
|
523
|
+
if (area === 'variants' && !parseName(area, name, board, variantIds)) fs.rmSync(path.join(dir, name), { force: true })
|
|
524
|
+
else files.add(`${area}/${name}`)
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
fs.rmSync(path.join(staging, 'checks'), { recursive: true, force: true })
|
|
528
|
+
if (keptReports.length) {
|
|
529
|
+
fs.mkdirSync(path.join(staging, 'checks'), { recursive: true })
|
|
530
|
+
for (const r of keptReports) fs.writeFileSync(path.join(staging, 'checks', `${r.id}.json`), JSON.stringify(r))
|
|
531
|
+
}
|
|
532
|
+
fs.mkdirSync(path.join(staging, 'captures'), { recursive: true })
|
|
533
|
+
fs.writeFileSync(path.join(staging, 'captures', 'manifest.json'), JSON.stringify(cleanManifest(manifest, board, viewports, files)))
|
|
534
|
+
fs.writeFileSync(path.join(staging, 'variants.json'), JSON.stringify({ variants: cleanList }))
|
|
535
|
+
const target = path.join(projectDir(dataDir, t.workspaceId, projectId), board.id)
|
|
536
|
+
const old = `${target}.old-${push.id}`
|
|
537
|
+
fs.mkdirSync(path.dirname(target), { recursive: true })
|
|
538
|
+
transaction(db, () => {
|
|
539
|
+
const at = now.toISOString()
|
|
540
|
+
const existing = t.get('SELECT board_id FROM boards WHERE workspace_id = :ws AND project_id = :project AND board_id = :board', {
|
|
541
|
+
project: projectId,
|
|
542
|
+
board: board.id,
|
|
543
|
+
})
|
|
544
|
+
// Rechecked here: other pushes may have committed since this one started.
|
|
545
|
+
if (!existing) {
|
|
546
|
+
const count = t.get('SELECT COUNT(*) AS n FROM boards WHERE workspace_id = :ws AND project_id = :project', { project: projectId }).n
|
|
547
|
+
if (count >= boardsPerProject) throw new PushError(409, 'too_many_boards', `A project holds at most ${boardsPerProject} boards.`)
|
|
548
|
+
}
|
|
549
|
+
const bytes = push.bytes + checkBytes
|
|
550
|
+
const others = t.get(
|
|
551
|
+
'SELECT COALESCE(SUM(bytes), 0) AS n FROM boards WHERE workspace_id = :ws AND NOT (project_id = :project AND board_id = :board)',
|
|
552
|
+
{ project: projectId, board: board.id },
|
|
553
|
+
).n
|
|
554
|
+
if (others + bytes > workspaceQuota) throw new PushError(413, 'quota_exceeded', 'This workspace is over its storage quota.')
|
|
555
|
+
const definition = JSON.stringify(push.definition)
|
|
556
|
+
if (existing) {
|
|
557
|
+
t.run(
|
|
558
|
+
'UPDATE boards SET definition = :definition, pushed_at = :at, push_count = push_count + 1, bytes = :bytes WHERE workspace_id = :ws AND project_id = :project AND board_id = :board',
|
|
559
|
+
{ definition, at, bytes, project: projectId, board: board.id },
|
|
560
|
+
)
|
|
561
|
+
} else {
|
|
562
|
+
t.run(
|
|
563
|
+
`INSERT INTO boards (workspace_id, project_id, board_id, definition, pushed_at, push_count, bytes, created_at)
|
|
564
|
+
VALUES (:ws, :project, :board, :definition, :at, 1, :bytes, :at)`,
|
|
565
|
+
{ project: projectId, board: board.id, definition, at, bytes },
|
|
566
|
+
)
|
|
567
|
+
}
|
|
568
|
+
t.run("UPDATE pushes SET status = 'committed', finished_at = :at WHERE workspace_id = :ws AND id = :id", { id: push.id, at })
|
|
569
|
+
// Inside the transaction: if the swap fails, the rows roll back too.
|
|
570
|
+
if (fs.existsSync(target)) fs.renameSync(target, old)
|
|
571
|
+
try {
|
|
572
|
+
fs.renameSync(staging, target)
|
|
573
|
+
} catch (error) {
|
|
574
|
+
if (fs.existsSync(old)) fs.renameSync(old, target)
|
|
575
|
+
throw error
|
|
576
|
+
}
|
|
577
|
+
})
|
|
578
|
+
fs.rmSync(old, { recursive: true, force: true })
|
|
579
|
+
return { board: board.id, files: files.size, pages: board.pages.length, checks: cleanChecks.reports.length }
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
return { start, receive, commit, get, expire }
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
// The whole data folder, at start and then every few minutes: open pushes
|
|
586
|
+
// past their hour fail and lose their staging folder, staging folders with
|
|
587
|
+
// no open push (a crash mid-push) go, and so do board folders a commit left
|
|
588
|
+
// behind (`<board>.old-<push>`).
|
|
589
|
+
export function sweepStaging({ db, dataDir, now = new Date() }) {
|
|
590
|
+
const cutoff = new Date(now.getTime() - PUSH_LIMITS.openMs).toISOString()
|
|
591
|
+
let removed = 0
|
|
592
|
+
for (const row of staleOpenPushes(db, cutoff)) {
|
|
593
|
+
createPushStore(tenant(db, row.workspace_id), { db, dataDir }).expire(row.id, now)
|
|
594
|
+
removed += 1
|
|
595
|
+
}
|
|
596
|
+
const staging = path.join(dataDir, 'staging')
|
|
597
|
+
const open = openPushIds(db)
|
|
598
|
+
for (const name of fs.existsSync(staging) ? fs.readdirSync(staging) : []) {
|
|
599
|
+
if (open.has(name)) continue
|
|
600
|
+
fs.rmSync(path.join(staging, name), { recursive: true, force: true })
|
|
601
|
+
removed += 1
|
|
602
|
+
}
|
|
603
|
+
const files = path.join(dataDir, 'files')
|
|
604
|
+
for (const ws of fs.existsSync(files) ? fs.readdirSync(files) : []) {
|
|
605
|
+
for (const project of safeList(path.join(files, ws))) {
|
|
606
|
+
for (const name of safeList(path.join(files, ws, project))) {
|
|
607
|
+
if (!/\.old-ps_[a-f0-9]+$/.test(name)) continue
|
|
608
|
+
fs.rmSync(path.join(files, ws, project, name), { recursive: true, force: true })
|
|
609
|
+
removed += 1
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
return removed
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
function safeList(dir) {
|
|
617
|
+
try {
|
|
618
|
+
return fs.readdirSync(dir)
|
|
619
|
+
} catch {
|
|
620
|
+
return []
|
|
621
|
+
}
|
|
622
|
+
}
|