shapewall 0.2.4 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +45 -0
- package/README.md +35 -1
- package/dist/agent/package-lock.json +2 -2
- package/dist/client/assets/BoardChat-V1BCXdDB.js +27 -0
- package/dist/client/assets/{BuzzDialog-CZdwIQLF.js → BuzzDialog-Ds7Thpfe.js} +1 -1
- package/dist/client/assets/ChecksPanel-Cm22bCdQ.js +1 -0
- package/dist/client/assets/History-U0bD1BjA.js +1 -0
- package/dist/client/assets/{LocalSettings-B5gtfKEl.js → LocalSettings-DOSuW7VC.js} +1 -1
- package/dist/client/assets/{Management-BxEUZAmI.js → Management-BFcKFdbp.js} +1 -1
- package/dist/client/assets/QuickConnect-DyfPSWbV.js +1 -0
- package/dist/client/assets/{ShareDialog-CdVAreaw.js → ShareDialog-BouvNraR.js} +2 -2
- package/dist/client/assets/{app-update-dialog-BONq9YFp.js → app-update-dialog-DH2Mgby8.js} +1 -1
- package/dist/client/assets/{circle-x-BmT6vHKl.js → circle-x-PRxj7kPi.js} +1 -1
- package/dist/client/assets/index-C9S7HL_o.js +37 -0
- package/dist/client/assets/index-CXtuJi1K.css +2 -0
- package/dist/client/assets/{refresh-cw-ugt3cWKH.js → refresh-cw-aeOBDuGQ.js} +1 -1
- package/dist/client/assets/{square-_KG6tM5A.js → shield-check-DuPBJ6H_.js} +1 -1
- package/dist/client/assets/ui-Cg4TTrec.js +41 -0
- package/dist/client/build-info.json +1 -1
- package/dist/client/index.html +3 -3
- package/dist/client/sw.js +2 -2
- package/dist/console/assets/LocalAccountsApp-lJjkhUcH.js +1 -0
- package/dist/console/assets/LocalConsole-BZHqYKnt.js +1 -0
- package/dist/console/assets/{app-update-dialog-o4QY_jL9.js → app-update-dialog-DCM9sYE3.js} +1 -1
- package/dist/console/assets/console-C2LsvEco.js +55 -0
- package/dist/console/assets/console-Fp4k9iyx.css +2 -0
- package/dist/console/build-info.json +1 -1
- package/dist/console/console.html +2 -2
- package/docs/agent-hardening.md +19 -0
- package/docs/agent-workflow.md +33 -0
- package/docs/agents.md +112 -0
- package/docs/config.md +41 -0
- package/docs/deployment.md +3 -0
- package/docs/hosted.md +10 -0
- package/docs/pull-requests.md +193 -0
- package/package.json +2 -1
- package/src/agent/backends/acp.js +23 -6
- package/src/agent/batch-fixes.js +45 -0
- package/src/agent/batch-queue.js +204 -0
- package/src/agent/branch-mode.js +753 -0
- package/src/agent/config.js +6 -0
- package/src/agent/fix-prompt.js +5 -0
- package/src/agent/routes.js +850 -185
- package/src/agent/run.js +8 -1
- package/src/agent/settings-routes.js +69 -1
- package/src/agent/starter/worktree.js +286 -29
- package/src/agent/threads.js +2 -7
- package/src/agent/tools.js +51 -0
- package/src/agent/user-settings.js +33 -2
- package/src/ci/capture.js +103 -0
- package/src/ci/compare.js +267 -0
- package/src/ci/github.js +307 -0
- package/src/ci/images.js +76 -0
- package/src/ci/settings.js +211 -0
- package/src/ci/start.js +148 -0
- package/src/ci/summary.js +262 -0
- package/src/ci/workflow.js +249 -0
- package/src/cli/args.js +24 -1
- package/src/cli/ci.js +229 -0
- package/src/cli/help.js +23 -1
- package/src/cli/init.js +36 -1
- package/src/cli.js +6 -2
- package/src/comment-drawing.js +161 -0
- package/src/comment-fields.js +51 -0
- package/src/comment-media.js +528 -0
- package/src/config.js +7 -0
- package/src/doctor.js +23 -0
- package/src/export.js +8 -0
- package/src/feedback.js +89 -12
- package/src/github.js +3 -3
- package/src/hosted/admin.js +36 -4
- package/src/hosted/app.js +5 -0
- package/src/hosted/board-routes.js +63 -7
- package/src/hosted/clean-worker.js +22 -4
- package/src/hosted/cleaner.js +4 -1
- package/src/hosted/comment-files.js +200 -0
- package/src/hosted/comments.js +68 -15
- package/src/hosted/db.js +32 -0
- package/src/hosted/guest-view.js +9 -0
- package/src/hosted/import.js +213 -18
- package/src/hosted/operator.js +3 -1
- package/src/hosted/projects.js +3 -2
- package/src/hosted/push.js +6 -3
- package/src/hosted/retention.js +18 -0
- package/src/hosted/tenancy.js +1 -0
- package/src/hosted/token-api.js +110 -9
- package/src/image-sniff.js +11 -0
- package/src/mcp.js +48 -3
- package/src/remote/remote-mcp.js +2 -2
- package/src/remote/remote.js +6 -2
- package/src/remote/share.js +77 -7
- package/src/server/app.js +18 -1
- package/src/store.js +119 -33
- package/src/watch.js +4 -0
- package/dist/client/assets/BoardChat-C0a3DDjv.js +0 -28
- package/dist/client/assets/ChecksPanel-BHWgwJWO.js +0 -1
- package/dist/client/assets/History-C01qLFpP.js +0 -1
- package/dist/client/assets/QuickConnect-BzTM3UNO.js +0 -1
- package/dist/client/assets/index-C1IsGdGs.js +0 -29
- package/dist/client/assets/index-DnsWelgZ.css +0 -2
- package/dist/client/assets/ui-BAGBTB_P.js +0 -41
- package/dist/console/assets/LocalAccountsApp-DtfTDbQg.js +0 -1
- package/dist/console/assets/LocalConsole-BPpsxDq_.js +0 -1
- package/dist/console/assets/console-DeOji9Zv.css +0 -2
- package/dist/console/assets/console-MvFrlUCC.js +0 -51
package/src/feedback.js
CHANGED
|
@@ -9,15 +9,71 @@ import path from 'node:path'
|
|
|
9
9
|
import { quoteData } from './agent/context.js'
|
|
10
10
|
import { boardViewports, frameSize, getBoard, joinUrl } from './config.js'
|
|
11
11
|
import { boardDir, entryVersion, listComments, readManifest, readVariants, resolveFile } from './store.js'
|
|
12
|
+
import { commentFilePath, markupFile } from './comment-media.js'
|
|
12
13
|
|
|
13
14
|
// What the items of one board need to know about its pages and variants.
|
|
14
|
-
|
|
15
|
+
// `media` finds a comment's files: imagePath(image) and markupPath({
|
|
16
|
+
// commentId, replyId, drawing, screenshot }) give a path or null. The local
|
|
17
|
+
// store's by default; the hosted server passes its own.
|
|
18
|
+
export function feedbackContext(config, board, { media = null } = {}) {
|
|
19
|
+
const folder = boardDir(config, board.id)
|
|
15
20
|
return {
|
|
16
21
|
config,
|
|
17
22
|
board,
|
|
18
23
|
pages: new Map(board.pages.map((page) => [page.id, page])),
|
|
19
24
|
variants: new Map(readVariants(config, board.id).map((variant) => [`variant:${variant.id}`, variant])),
|
|
20
25
|
manifest: readManifest(config, board.id),
|
|
26
|
+
media: media || {
|
|
27
|
+
imagePath: (image) => commentFilePath(folder, image.file),
|
|
28
|
+
markupPath: (input) => markupFile(folder, input),
|
|
29
|
+
},
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// The images of a comment or reply as agents get them: the file, its size
|
|
34
|
+
// and type (the name is the uploader's, untrusted like the text).
|
|
35
|
+
function imageFiles(context, images) {
|
|
36
|
+
return (images || [])
|
|
37
|
+
.map((image) => ({
|
|
38
|
+
name: image.name,
|
|
39
|
+
path: context.media.imagePath(image),
|
|
40
|
+
mediaType: image.mediaType,
|
|
41
|
+
bytes: image.bytes,
|
|
42
|
+
width: image.width,
|
|
43
|
+
height: image.height,
|
|
44
|
+
}))
|
|
45
|
+
.filter((image) => image.path)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// The screenshot a drawing was made on: that capture version's, or the
|
|
49
|
+
// latest when it is the latest (or the version is not kept).
|
|
50
|
+
function drawnOn(context, comment, page, name, kind, drawing) {
|
|
51
|
+
const { config, board } = context
|
|
52
|
+
if (!name) return null
|
|
53
|
+
const latest = resolveFile(config, board.id, kind, `${name}.${comment.viewport}.png`)
|
|
54
|
+
const current = page && latest ? entryVersion(context.manifest?.pages?.[page.id]?.[comment.viewport], context.manifest) : null
|
|
55
|
+
const made = drawing?.captureVersion
|
|
56
|
+
if (page && made && made !== current) {
|
|
57
|
+
return { file: resolveFile(config, board.id, 'versions', `${name}.${comment.viewport}.${made}.png`), previous: true }
|
|
58
|
+
}
|
|
59
|
+
return { file: latest, previous: false }
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function drawingSummary(context, comment, page, name, kind, drawing, replyId = null) {
|
|
63
|
+
if (!drawing) return { drawing: null, markup: null }
|
|
64
|
+
const on = drawnOn(context, comment, page, name, kind, drawing)
|
|
65
|
+
const markup = on?.file
|
|
66
|
+
? context.media.markupPath({ commentId: comment.id, replyId, drawing, screenshot: on.file, kind: on.previous ? 'versions' : kind })
|
|
67
|
+
: null
|
|
68
|
+
return {
|
|
69
|
+
drawing: {
|
|
70
|
+
shapes: drawing.shapes.length,
|
|
71
|
+
tools: [...new Set(drawing.shapes.map((shape) => shape.tool))],
|
|
72
|
+
captureVersion: drawing.captureVersion || null,
|
|
73
|
+
// Drawn on a capture the page has been captured again since.
|
|
74
|
+
onPreviousCapture: Boolean(on?.previous),
|
|
75
|
+
},
|
|
76
|
+
markup,
|
|
21
77
|
}
|
|
22
78
|
}
|
|
23
79
|
|
|
@@ -71,6 +127,7 @@ export function feedbackItem(context, comment) {
|
|
|
71
127
|
const size = frame ? { width: frame.width, height: frame.height } : { width: 0, height: 0 }
|
|
72
128
|
const name = page ? page.id : variant?.id
|
|
73
129
|
const kind = page ? 'captures' : 'variants'
|
|
130
|
+
const drawn = drawingSummary(context, comment, page, name, kind, comment.drawing)
|
|
74
131
|
return {
|
|
75
132
|
id: comment.id,
|
|
76
133
|
status: comment.status,
|
|
@@ -92,6 +149,8 @@ export function feedbackItem(context, comment) {
|
|
|
92
149
|
fixRef: comment.fixRef || null,
|
|
93
150
|
agentStatus: comment.agentStatus || null,
|
|
94
151
|
agentStatusAt: comment.agentStatusAt || null,
|
|
152
|
+
// branch-ready: the branch the agent's fix waits on (agent branch mode).
|
|
153
|
+
...(comment.agentBranch ? { agentBranch: comment.agentBranch } : {}),
|
|
95
154
|
// Handles mentioned in the comment or its replies ("@robin" -> "robin").
|
|
96
155
|
mentions: comment.mentions || [],
|
|
97
156
|
reopenCount: comment.reopenCount || 0,
|
|
@@ -99,15 +158,23 @@ export function feedbackItem(context, comment) {
|
|
|
99
158
|
editedAt: comment.editedAt || null,
|
|
100
159
|
// The GitHub issue made from it and the pull requests linked to it.
|
|
101
160
|
github: comment.github || { issue: null, pulls: [] },
|
|
102
|
-
replies: (comment.replies || []).map((reply) =>
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
161
|
+
replies: (comment.replies || []).map((reply) => {
|
|
162
|
+
const drawn = drawingSummary(context, comment, page, name, kind, reply.drawing, reply.id)
|
|
163
|
+
const images = imageFiles(context, reply.images)
|
|
164
|
+
return {
|
|
165
|
+
id: reply.id,
|
|
166
|
+
author: reply.author,
|
|
167
|
+
...(reply.guest ? { guest: true } : {}),
|
|
168
|
+
...(reply.handle ? { handle: reply.handle } : {}),
|
|
169
|
+
...(reply.agent ? { agent: true } : {}),
|
|
170
|
+
body: reply.body,
|
|
171
|
+
createdAt: reply.createdAt,
|
|
172
|
+
// Only when the reply has them: images (paths) and a drawing with
|
|
173
|
+
// its rendered markup PNG.
|
|
174
|
+
...(images.length ? { images } : {}),
|
|
175
|
+
...(drawn.drawing ? { drawing: drawn.drawing, markup: drawn.markup } : {}),
|
|
176
|
+
}
|
|
177
|
+
}),
|
|
111
178
|
page: page
|
|
112
179
|
? {
|
|
113
180
|
id: page.id,
|
|
@@ -130,7 +197,10 @@ export function feedbackItem(context, comment) {
|
|
|
130
197
|
// screenshot / liveHtml: what the comment was made on (version: that
|
|
131
198
|
// capture version; the latest capture when that one is gone).
|
|
132
199
|
// latestScreenshot / latestLiveHtml / latestVersion: the page as it is now.
|
|
133
|
-
|
|
200
|
+
// images: what the reviewer attached ({ name, path, mediaType, bytes,
|
|
201
|
+
// width, height }); markup: the drawing on its screenshot as a PNG.
|
|
202
|
+
drawing: drawn.drawing,
|
|
203
|
+
files: { ...commentFiles(context, comment, page, name, kind), images: imageFiles(context, comment.images), markup: drawn.markup },
|
|
134
204
|
}
|
|
135
205
|
}
|
|
136
206
|
|
|
@@ -157,7 +227,9 @@ export function formatFeedbackItem(item, { cwd = process.cwd() } = {}) {
|
|
|
157
227
|
const rel = (file) => (file ? (/^https?:\/\//.test(file) ? file : path.relative(cwd, file) || file) : 'not captured')
|
|
158
228
|
const lines = []
|
|
159
229
|
const where = item.page ? `${item.page.title} (${item.page.path})` : `Variant: ${item.variant?.title || 'removed'}`
|
|
160
|
-
const agent = item.agentStatus
|
|
230
|
+
const agent = item.agentStatus
|
|
231
|
+
? ` [agent: ${item.agentStatus}${item.agentStatus === 'branch-ready' && item.agentBranch ? ` on ${item.agentBranch}` : ''}]`
|
|
232
|
+
: ''
|
|
161
233
|
const reopened = item.reopenCount && item.status === 'open' ? ' [reopened]' : ''
|
|
162
234
|
const guest = item.authorGuest ? ' [guest]' : ''
|
|
163
235
|
// A guest's name gets its own warning line below.
|
|
@@ -172,6 +244,9 @@ export function formatFeedbackItem(item, { cwd = process.cwd() } = {}) {
|
|
|
172
244
|
lines.push(` at: ${item.position.x},${item.position.y} px (${item.position.xPercent}% / ${item.position.yPercent}%)`)
|
|
173
245
|
if (item.page) lines.push(` url: ${item.page.url}`)
|
|
174
246
|
lines.push(` screenshot: ${rel(item.files.screenshot)}`)
|
|
247
|
+
for (const image of item.files.images || []) lines.push(` image: ${rel(image.path)} (${image.mediaType}, ${image.bytes} bytes)`)
|
|
248
|
+
if (item.files.markup)
|
|
249
|
+
lines.push(` drawing: ${rel(item.files.markup)}${item.drawing?.onPreviousCapture ? ' (on the previous capture)' : ''}`)
|
|
175
250
|
if (item.files.latestScreenshot && item.files.latestScreenshot !== item.files.screenshot) {
|
|
176
251
|
lines.push(` latest screenshot: ${rel(item.files.latestScreenshot)} (captured again since the comment)`)
|
|
177
252
|
}
|
|
@@ -183,6 +258,8 @@ export function formatFeedbackItem(item, { cwd = process.cwd() } = {}) {
|
|
|
183
258
|
const [first, ...rest] = reply.body.split('\n')
|
|
184
259
|
lines.push(` > ${reply.author}${reply.guest ? ' [guest]' : ''}: ${first}`)
|
|
185
260
|
for (const line of rest) lines.push(` > ${line}`)
|
|
261
|
+
for (const image of reply.images || []) lines.push(` > image: ${rel(image.path)}`)
|
|
262
|
+
if (reply.markup) lines.push(` > drawing: ${rel(reply.markup)}`)
|
|
186
263
|
}
|
|
187
264
|
return lines
|
|
188
265
|
}
|
package/src/github.js
CHANGED
|
@@ -24,7 +24,7 @@ import { absoluteProgram } from './which.js'
|
|
|
24
24
|
// Checked in this order; the first one set wins.
|
|
25
25
|
export const TOKEN_VARS = ['SHAPEWALL_GITHUB_TOKEN', 'GITHUB_TOKEN']
|
|
26
26
|
export const WEBHOOK_SECRET_VAR = 'SHAPEWALL_GITHUB_WEBHOOK_SECRET'
|
|
27
|
-
const DEFAULT_API_URL = 'https://api.github.com'
|
|
27
|
+
export const DEFAULT_API_URL = 'https://api.github.com'
|
|
28
28
|
const REQUEST_TIMEOUT_MS = 20000
|
|
29
29
|
const SYNC_PAGES_MAX = 5
|
|
30
30
|
const SYNC_PAGE_SIZE = 50
|
|
@@ -41,7 +41,7 @@ const LOOPBACK = new Set(['127.0.0.1', 'localhost', '[::1]', '::1'])
|
|
|
41
41
|
// person named themselves ($SHAPEWALL_GITHUB_API_URL). An API URL that only a
|
|
42
42
|
// config file names gets no token: a cloned repository's config could
|
|
43
43
|
// otherwise point it at any host.
|
|
44
|
-
function tokenFrom(env, apiUrl = DEFAULT_API_URL, confirmed = true) {
|
|
44
|
+
export function tokenFrom(env, apiUrl = DEFAULT_API_URL, confirmed = true) {
|
|
45
45
|
if (apiUrl !== DEFAULT_API_URL && !confirmed) return null
|
|
46
46
|
for (const name of TOKEN_VARS) {
|
|
47
47
|
const value = typeof env[name] === 'string' ? env[name].trim() : ''
|
|
@@ -86,7 +86,7 @@ export function ghToken(env = process.env, { run = execFileSync, now = Date.now(
|
|
|
86
86
|
|
|
87
87
|
// The API must be https; plain http only on this machine (a test stub or a
|
|
88
88
|
// local proxy), so the token never crosses a network in the clear.
|
|
89
|
-
function checkApiUrl(value) {
|
|
89
|
+
export function checkApiUrl(value) {
|
|
90
90
|
try {
|
|
91
91
|
const url = new URL(value)
|
|
92
92
|
if (url.protocol === 'https:') return null
|
package/src/hosted/admin.js
CHANGED
|
@@ -105,7 +105,18 @@ export function deleteWorkspaceData(db, t, dataDir) {
|
|
|
105
105
|
// files, to run once the transaction committed.
|
|
106
106
|
export function deleteWorkspaceRows(t, dataDir) {
|
|
107
107
|
const openPushes = t.all("SELECT id FROM pushes WHERE workspace_id = :ws AND status = 'open'").map((row) => row.id)
|
|
108
|
-
for (const table of [
|
|
108
|
+
for (const table of [
|
|
109
|
+
'comment_files',
|
|
110
|
+
'comments',
|
|
111
|
+
'boards',
|
|
112
|
+
'pushes',
|
|
113
|
+
'api_tokens',
|
|
114
|
+
'shares',
|
|
115
|
+
'invites',
|
|
116
|
+
'projects',
|
|
117
|
+
'memberships',
|
|
118
|
+
'audit_log',
|
|
119
|
+
]) {
|
|
109
120
|
t.run(`DELETE FROM ${table} WHERE workspace_id = :ws`)
|
|
110
121
|
}
|
|
111
122
|
t.run('DELETE FROM workspaces WHERE workspace_id = :ws')
|
|
@@ -213,7 +224,14 @@ export function planWorkspaceExport(t, { dataDir, projects, iso }) {
|
|
|
213
224
|
total += size
|
|
214
225
|
return true
|
|
215
226
|
})
|
|
216
|
-
|
|
227
|
+
// Images on its comments (in the database, src/hosted/comment-files.js).
|
|
228
|
+
const images = t.all(
|
|
229
|
+
`SELECT file, bytes FROM comment_files WHERE workspace_id = :ws AND project_id = :project AND board_id = :board
|
|
230
|
+
AND comment_number IS NOT NULL ORDER BY file`,
|
|
231
|
+
{ project: project.id, board: row.board_id },
|
|
232
|
+
)
|
|
233
|
+
for (const image of images) total += image.bytes
|
|
234
|
+
plan.push({ project, row, files, images })
|
|
217
235
|
}
|
|
218
236
|
}
|
|
219
237
|
const members = t
|
|
@@ -251,7 +269,7 @@ export function planWorkspaceExport(t, { dataDir, projects, iso }) {
|
|
|
251
269
|
projects,
|
|
252
270
|
plan,
|
|
253
271
|
total,
|
|
254
|
-
files: plan.reduce((sum, item) => sum + item.files.length, 0),
|
|
272
|
+
files: plan.reduce((sum, item) => sum + item.files.length + item.images.length, 0),
|
|
255
273
|
members,
|
|
256
274
|
tokens,
|
|
257
275
|
shares,
|
|
@@ -306,7 +324,7 @@ export async function writeWorkspaceExport(t, exported, { write, now, iso, expor
|
|
|
306
324
|
}
|
|
307
325
|
await audit.write(count ? '\n]\n' : ']\n')
|
|
308
326
|
await audit.end()
|
|
309
|
-
for (const { project, row, files } of plan) {
|
|
327
|
+
for (const { project, row, files, images = [] } of plan) {
|
|
310
328
|
const base = `${root}projects/${project.slug}/boards/${row.board_id}/`
|
|
311
329
|
const comments = createCommentStore(t, project.id, row.board_id).list()
|
|
312
330
|
await zip.add(
|
|
@@ -325,6 +343,20 @@ export async function writeWorkspaceExport(t, exported, { write, now, iso, expor
|
|
|
325
343
|
}
|
|
326
344
|
await zip.add(`${base}files/${relative}`, data)
|
|
327
345
|
}
|
|
346
|
+
// comments.json lists each image ({ file, name, ... }); the file is in
|
|
347
|
+
// comment-files/. One read at a time; gone since the plan: skipped.
|
|
348
|
+
for (const { file } of images) {
|
|
349
|
+
const stored = t.get(
|
|
350
|
+
`SELECT data FROM comment_files WHERE workspace_id = :ws AND project_id = :project AND board_id = :board AND file = :file
|
|
351
|
+
AND comment_number IS NOT NULL`,
|
|
352
|
+
{ project: project.id, board: row.board_id, file },
|
|
353
|
+
)
|
|
354
|
+
if (!stored) {
|
|
355
|
+
skipped.push(`${base.slice(root.length)}comment-files/${file}`)
|
|
356
|
+
continue
|
|
357
|
+
}
|
|
358
|
+
await zip.add(`${base}comment-files/${file}`, Buffer.from(stored.data), { compress: false })
|
|
359
|
+
}
|
|
328
360
|
}
|
|
329
361
|
// Files removed after the plan (a push swapped the board, retention pruned
|
|
330
362
|
// a version) are listed instead of cutting the zip short.
|
package/src/hosted/app.js
CHANGED
|
@@ -42,6 +42,7 @@ import { changeEventType, createEventBus, publishMutations } from './events.js'
|
|
|
42
42
|
import { mountMcpHttp } from './mcp-http.js'
|
|
43
43
|
import { createProjectStore } from './projects.js'
|
|
44
44
|
import { createCleaner } from './cleaner.js'
|
|
45
|
+
import { sweepStagedImages } from './comment-files.js'
|
|
45
46
|
import { sweepStaging } from './push.js'
|
|
46
47
|
import { createRateLimiter } from './ratelimit.js'
|
|
47
48
|
import { createQuotas } from './quotas.js'
|
|
@@ -391,6 +392,9 @@ export function createHostedApp({
|
|
|
391
392
|
policyFor,
|
|
392
393
|
dispatchView,
|
|
393
394
|
boardContext,
|
|
395
|
+
cleaner,
|
|
396
|
+
quotas,
|
|
397
|
+
workspaceQuota,
|
|
394
398
|
deps,
|
|
395
399
|
})
|
|
396
400
|
|
|
@@ -469,6 +473,7 @@ export function createHostedApp({
|
|
|
469
473
|
const sweep = () => {
|
|
470
474
|
try {
|
|
471
475
|
sweepStaging({ db, dataDir, now: now() })
|
|
476
|
+
sweepStagedImages(db, now())
|
|
472
477
|
} catch (error) {
|
|
473
478
|
log(`staging sweep: ${error.message}`)
|
|
474
479
|
}
|
|
@@ -8,7 +8,7 @@ import express from 'express'
|
|
|
8
8
|
import { boardSummary } from '../server/app.js'
|
|
9
9
|
import { listJourneys } from '../journeys.js'
|
|
10
10
|
import { MENTION_HANDLE_RE } from '../mentions.js'
|
|
11
|
-
import { resolveFile } from '../store.js'
|
|
11
|
+
import { resolveFile, validateCommentInput, validateCommentMedia } from '../store.js'
|
|
12
12
|
import { avatarUrl } from '../accounts/avatar.js'
|
|
13
13
|
import { checksView, readReport } from '../checks/store.js'
|
|
14
14
|
import { agentPolicy } from './agent-policy.js'
|
|
@@ -17,6 +17,7 @@ import { guestView, rowToComment } from './comments.js'
|
|
|
17
17
|
import { transaction } from './db.js'
|
|
18
18
|
import { publicView } from './guest-view.js'
|
|
19
19
|
import { HttpError, fail, guestLabel } from './http.js'
|
|
20
|
+
import { checkStorageQuota, cleanImages, commentListsFile, readCommentFile, sendImage } from './comment-files.js'
|
|
20
21
|
import { boardPermissions, canDeleteComment, canEditComment, guestPermissions } from './permissions.js'
|
|
21
22
|
import { boardConfig, createProjectStore } from './projects.js'
|
|
22
23
|
import { readPublic } from './public.js'
|
|
@@ -46,6 +47,11 @@ export function mountBoardRoutes(
|
|
|
46
47
|
policyFor,
|
|
47
48
|
dispatchView,
|
|
48
49
|
boardContext,
|
|
50
|
+
// Cleans comment images off the event loop (src/hosted/cleaner.js).
|
|
51
|
+
cleaner = null,
|
|
52
|
+
// The storage quota comment images count against.
|
|
53
|
+
quotas = null,
|
|
54
|
+
workspaceQuota = undefined,
|
|
49
55
|
// What readGuest needs (the shared deps of src/hosted/app.js).
|
|
50
56
|
deps,
|
|
51
57
|
},
|
|
@@ -65,10 +71,16 @@ export function mountBoardRoutes(
|
|
|
65
71
|
next()
|
|
66
72
|
})
|
|
67
73
|
boardApi.use(guardWrites)
|
|
68
|
-
|
|
74
|
+
// New comments and replies may carry images (base64, 4 of 5 MB): their
|
|
75
|
+
// body is read after the session and role checks, with its own limit
|
|
76
|
+
// (mediaBody); every other route keeps 64 kB.
|
|
77
|
+
const MEDIA_POST_RE = /^\/boards\/[^/]+\/comments(?:\/[^/]+\/replies)?\/?$/
|
|
78
|
+
const skipMedia = (parser) => (req, res, next) =>
|
|
79
|
+
req.method === 'POST' && MEDIA_POST_RE.test(req.path) ? next() : parser(req, res, next)
|
|
80
|
+
boardApi.use(skipMedia(express.json({ limit: '64kb' })))
|
|
69
81
|
// Per account: API calls in one bucket, a board's files in a larger one.
|
|
70
82
|
const sessionLimits = { api: limit('session', (req) => req.user.id), files: limit('sessionFiles', (req) => req.user.id) }
|
|
71
|
-
const BOARD_FILE_RE = /^\/boards\/[^/]+\/(?:captures|variants|versions|live|variant-live|elements|variant-elements)\//
|
|
83
|
+
const BOARD_FILE_RE = /^\/boards\/[^/]+\/(?:captures|variants|versions|live|variant-live|elements|variant-elements|comment-files)\//
|
|
72
84
|
boardApi.use(needSession, (req, res, next) =>
|
|
73
85
|
(req.method === 'GET' && BOARD_FILE_RE.test(req.path) ? sessionLimits.files : sessionLimits.api)(req, res, next),
|
|
74
86
|
)
|
|
@@ -143,7 +155,7 @@ export function mountBoardRoutes(
|
|
|
143
155
|
if (!req.project) return fail(res, 401, 'share_ended', 'This share link has ended.')
|
|
144
156
|
next()
|
|
145
157
|
})
|
|
146
|
-
shareApi.use(express.json({ limit: '16kb' }))
|
|
158
|
+
shareApi.use(skipMedia(express.json({ limit: '16kb' })))
|
|
147
159
|
|
|
148
160
|
// A public project's board API (src/hosted/public.js): no cookie, view only.
|
|
149
161
|
const publicApi = express.Router({ mergeParams: true })
|
|
@@ -184,6 +196,23 @@ export function mountBoardRoutes(
|
|
|
184
196
|
}
|
|
185
197
|
next()
|
|
186
198
|
}
|
|
199
|
+
// The body of a new comment or reply: members may attach images (4 of
|
|
200
|
+
// 5 MB as base64, about 28 MB); a guest's carries a drawing at most.
|
|
201
|
+
const memberMediaJson = express.json({ limit: '28mb' })
|
|
202
|
+
const guestMediaJson = express.json({ limit: '160kb' })
|
|
203
|
+
const mediaBody = (req, res, next) => (req.guest ? guestMediaJson : memberMediaJson)(req, res, next)
|
|
204
|
+
// A new comment's or reply's images and drawing, checked and the images
|
|
205
|
+
// cleaned (nothing stored yet). Guests of a share link draw, but never
|
|
206
|
+
// upload: their images are refused before anything is decoded.
|
|
207
|
+
async function mediaOf(req) {
|
|
208
|
+
const raw = req.body || {}
|
|
209
|
+
if (req.guest && raw.images !== undefined && raw.images !== null && !(Array.isArray(raw.images) && !raw.images.length))
|
|
210
|
+
throw new HttpError(403, 'guest_no_images', 'Guests on a share link cannot attach images.')
|
|
211
|
+
const media = validateCommentMedia(raw)
|
|
212
|
+
return { images: media.images.length ? await cleanImages(cleaner, media.images) : [], drawing: media.drawing }
|
|
213
|
+
}
|
|
214
|
+
const quotaOf = (req) => workspaceQuota ?? quotas?.of(req.t.workspaceId).storageBytes ?? Infinity
|
|
215
|
+
const imageBytes = (media) => media.images.reduce((sum, image) => sum + image.buffer.length, 0)
|
|
187
216
|
// Who a new comment or reply is from. A guest's name always carries
|
|
188
217
|
// "(guest)", so it never reads as a member's.
|
|
189
218
|
const authorOf = (req) =>
|
|
@@ -358,6 +387,21 @@ export function mountBoardRoutes(
|
|
|
358
387
|
)
|
|
359
388
|
router.all('/boards/:bid/capture', (req, res) => fail(res, 403, 'capture_disabled', 'Hosted boards are updated with shapewall push.'))
|
|
360
389
|
|
|
390
|
+
// An image on a comment or reply of this board: only a file that comment
|
|
391
|
+
// lists, as its stored image type, behind the same rule as the comment
|
|
392
|
+
// (members, the link's guests and a public project's visitors all see
|
|
393
|
+
// every comment of the board; guestView and publicView keep images).
|
|
394
|
+
// Same origin and never a document (nosniff, sandbox CSP), so no signed
|
|
395
|
+
// usercontent URL: an image cannot run script.
|
|
396
|
+
router.get(
|
|
397
|
+
'/boards/:bid/comment-files/:file',
|
|
398
|
+
withBoard((req, res, ctx) => {
|
|
399
|
+
const row = readCommentFile(req.t, req.project.id, ctx.board.id, req.params.file)
|
|
400
|
+
const comment = row ? rowToComment(ctx.comments.get(row.comment_number)) : null
|
|
401
|
+
if (!row || !commentListsFile(shown(req, comment), row)) return fail(res, 404, 'not_found', 'No such image.')
|
|
402
|
+
sendImage(res, row.data, row.media_type)
|
|
403
|
+
}),
|
|
404
|
+
)
|
|
361
405
|
router.get(
|
|
362
406
|
'/boards/:bid/comments',
|
|
363
407
|
withBoard((req, res, ctx) => {
|
|
@@ -376,9 +420,14 @@ export function mountBoardRoutes(
|
|
|
376
420
|
router.post(
|
|
377
421
|
'/boards/:bid/comments',
|
|
378
422
|
canComment,
|
|
379
|
-
|
|
423
|
+
mediaBody,
|
|
424
|
+
withBoard(async (req, res, ctx) => {
|
|
425
|
+
// The cheap checks first: a bad pin never costs an image decode.
|
|
426
|
+
validateCommentInput(ctx.config, ctx.board, req.body)
|
|
427
|
+
const media = await mediaOf(req)
|
|
380
428
|
const comment = transaction(db, () => {
|
|
381
|
-
|
|
429
|
+
checkStorageQuota(req.t, imageBytes(media), quotaOf(req))
|
|
430
|
+
const made = ctx.comments.create(ctx.config, ctx.board, req.body, authorOf(req), now(), media)
|
|
382
431
|
if (!req.guest) policyFor(req.t).autoVouch(req.project.id, ctx.board.id, ctx.comments.get(made.id), req.user.id, req.role)
|
|
383
432
|
if (req.guest) auditOf(req, 'guest_commented', `${ctx.board.id}#${made.id}`, { project: req.project.id, guest: req.guest.name })
|
|
384
433
|
return made
|
|
@@ -407,14 +456,21 @@ export function mountBoardRoutes(
|
|
|
407
456
|
router.post(
|
|
408
457
|
'/boards/:bid/comments/:cid/replies',
|
|
409
458
|
canComment,
|
|
410
|
-
|
|
459
|
+
mediaBody,
|
|
460
|
+
withBoard(async (req, res, ctx) => {
|
|
461
|
+
// The comment and the text first, then the images.
|
|
462
|
+
ctx.comments.get(req.params.cid)
|
|
463
|
+
if (typeof req.body?.body !== 'string' || !req.body.body.trim()) throw new HttpError(400, 'empty_comment', 'Write something first.')
|
|
464
|
+
const media = await mediaOf(req)
|
|
411
465
|
const result = transaction(db, () => {
|
|
466
|
+
checkStorageQuota(req.t, imageBytes(media), quotaOf(req))
|
|
412
467
|
const author = authorOf(req)
|
|
413
468
|
const made = ctx.comments.reply(
|
|
414
469
|
req.params.cid,
|
|
415
470
|
{ body: req.body?.body },
|
|
416
471
|
{ name: author.name, handle: author.handle, userId: author.userId, guest: author.guest, shareId: author.shareId },
|
|
417
472
|
now(),
|
|
473
|
+
{ ...media, config: ctx.config },
|
|
418
474
|
)
|
|
419
475
|
if (req.guest)
|
|
420
476
|
auditOf(req, 'guest_replied', `${ctx.board.id}#${made.comment.id}`, { project: req.project.id, guest: req.guest.name })
|
|
@@ -1,18 +1,36 @@
|
|
|
1
1
|
// Worker thread for cleaning pushed files (src/hosted/cleaner.js): decoding
|
|
2
2
|
// and re-encoding a PNG, or sanitizing live HTML, takes CPU time that must
|
|
3
|
-
// not block the server's event loop.
|
|
3
|
+
// not block the server's event loop. Comment images ('comment-image', base
|
|
4
|
+
// = the media type) and markup renders ('markup', base = the drawing as
|
|
5
|
+
// JSON, body = the screenshot) run here too.
|
|
4
6
|
|
|
5
7
|
import { parentPort } from 'node:worker_threads'
|
|
8
|
+
import { CommentError } from '../comment-fields.js'
|
|
9
|
+
import { cleanCommentImage, renderMarkup } from '../comment-media.js'
|
|
6
10
|
import { cleanHtml, cleanPng, PushError } from './push.js'
|
|
7
11
|
|
|
12
|
+
function run(kind, buffer, base) {
|
|
13
|
+
if (kind === 'html') return { clean: cleanHtml(buffer, { baseUrl: base }) }
|
|
14
|
+
if (kind === 'comment-image') {
|
|
15
|
+
const image = cleanCommentImage(buffer, base)
|
|
16
|
+
return { clean: image.buffer, meta: { mediaType: image.mediaType, width: image.width, height: image.height } }
|
|
17
|
+
}
|
|
18
|
+
if (kind === 'markup') {
|
|
19
|
+
const rendered = renderMarkup(buffer, JSON.parse(base))
|
|
20
|
+
if (!rendered) throw new PushError(422, 'no_markup', 'The drawing could not be rendered.')
|
|
21
|
+
return { clean: rendered.png }
|
|
22
|
+
}
|
|
23
|
+
return { clean: cleanPng(buffer) }
|
|
24
|
+
}
|
|
25
|
+
|
|
8
26
|
parentPort.on('message', ({ id, kind, body, base }) => {
|
|
9
27
|
try {
|
|
10
28
|
const buffer = Buffer.from(body.buffer, body.byteOffset, body.byteLength)
|
|
11
|
-
const clean = kind
|
|
29
|
+
const { clean, meta } = run(kind, buffer, base)
|
|
12
30
|
const out = new Uint8Array(clean.buffer, clean.byteOffset, clean.byteLength)
|
|
13
|
-
parentPort.postMessage({ id, ok: true, body: out })
|
|
31
|
+
parentPort.postMessage({ id, ok: true, body: out, ...(meta ? { meta } : {}) })
|
|
14
32
|
} catch (error) {
|
|
15
|
-
const known = error instanceof PushError
|
|
33
|
+
const known = error instanceof PushError || error instanceof CommentError
|
|
16
34
|
parentPort.postMessage({
|
|
17
35
|
id,
|
|
18
36
|
ok: false,
|
package/src/hosted/cleaner.js
CHANGED
|
@@ -46,7 +46,10 @@ export function createCleaner({ size = 2, maxQueue = 32, timeoutMs = 30_000, mem
|
|
|
46
46
|
clearTimeout(job.timer)
|
|
47
47
|
busy.delete(worker)
|
|
48
48
|
idle.push(worker)
|
|
49
|
-
|
|
49
|
+
// A comment image comes back with what was read from it: { buffer,
|
|
50
|
+
// mediaType, width, height }. Everything else: the Buffer.
|
|
51
|
+
const buffer = message.ok ? Buffer.from(message.body.buffer, message.body.byteOffset, message.body.byteLength) : null
|
|
52
|
+
if (message.ok) job.resolve(message.meta ? { ...message.meta, buffer } : buffer)
|
|
50
53
|
else job.reject(new PushError(message.status, message.code, message.message))
|
|
51
54
|
pump()
|
|
52
55
|
}
|