shapewall 0.0.0-stage → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +139 -0
- package/Dockerfile.agent +45 -0
- package/LICENSE +202 -0
- package/NOTICE +12 -0
- package/README.md +651 -2
- package/THIRD_PARTY.md +165 -0
- package/bin/shapewall.js +29 -0
- package/dist/agent/package-lock.json +8384 -0
- package/dist/client/assets/BoardChat-D5SFd8HE.js +28 -0
- package/dist/client/assets/BuzzDialog-HWXBy52W.js +1 -0
- package/dist/client/assets/ChecksPanel-DTdImWoD.js +1 -0
- package/dist/client/assets/History-CEv6b-av.js +1 -0
- package/dist/client/assets/LocalSettings-C4vriEVQ.js +1 -0
- package/dist/client/assets/Management-DCfhcEUY.js +1 -0
- package/dist/client/assets/QuickConnect-DQGgUsl1.js +1 -0
- package/dist/client/assets/ShareDialog-C_plIto0.js +2 -0
- package/dist/client/assets/app-update-dialog-2hPz_N9G.js +1 -0
- package/dist/client/assets/circle-x-BZMPi8Zf.js +1 -0
- package/dist/client/assets/index-ByQRKVit.css +2 -0
- package/dist/client/assets/index-DweQ8Qtu.js +29 -0
- package/dist/client/assets/refresh-cw-CRow0Nbq.js +1 -0
- package/dist/client/assets/runtime-Bft6x_6E.js +4 -0
- package/dist/client/assets/square-A8EnIYLR.js +1 -0
- package/dist/client/assets/tts-dist-DYlBQmwG.js +18 -0
- package/dist/client/assets/tts-en-g2p-BPUtBFGG.js +1 -0
- package/dist/client/assets/tts-ort-wasm-simd-threaded-qh98TCA_.wasm +0 -0
- package/dist/client/assets/tts-ort.wasm.bundle.min-rEkcs_ow.js +12 -0
- package/dist/client/assets/tts-rolldown-runtime-D8NudN84.js +1 -0
- package/dist/client/assets/tts-spacy-en-tokenizer-6SNMl--A.json +1 -0
- package/dist/client/assets/tts-tts.worker-Bzm6aWbv.js +1 -0
- package/dist/client/assets/tts-us_gold-DcjeNMH1.json +92589 -0
- package/dist/client/assets/tts-us_silver-C964mEB5.json +93363 -0
- package/dist/client/assets/ui-CLbws0rx.js +41 -0
- package/dist/client/build-info.json +1 -0
- package/dist/client/favicon.svg +7 -0
- package/dist/client/icons/apple-touch-icon.png +0 -0
- package/dist/client/icons/icon-192.png +0 -0
- package/dist/client/icons/icon-512.png +0 -0
- package/dist/client/icons/maskable-512.png +0 -0
- package/dist/client/index.html +25 -0
- package/dist/client/manifest.webmanifest +15 -0
- package/dist/client/sw.js +53 -0
- package/dist/client/theme-boot.js +27 -0
- package/dist/console/assets/LocalAccountsApp-BQCcmj3x.js +1 -0
- package/dist/console/assets/LocalConsole-Bb2FUDs7.js +1 -0
- package/dist/console/assets/app-update-dialog-DIeYepLZ.js +1 -0
- package/dist/console/assets/console-C7uJLLOn.js +51 -0
- package/dist/console/assets/console-R8IBY3Am.css +2 -0
- package/dist/console/build-info.json +1 -0
- package/dist/console/console.html +21 -0
- package/dist/console/favicon.svg +7 -0
- package/dist/console/icons/apple-touch-icon.png +0 -0
- package/dist/console/icons/icon-192.png +0 -0
- package/dist/console/icons/icon-512.png +0 -0
- package/dist/console/icons/maskable-512.png +0 -0
- package/dist/console/manifest.webmanifest +15 -0
- package/dist/console/theme-boot.js +27 -0
- package/docs/agent-hardening.md +342 -0
- package/docs/agent-recipes.md +187 -0
- package/docs/agent-workflow.md +320 -0
- package/docs/agents.md +894 -0
- package/docs/architecture.md +242 -0
- package/docs/buzz.md +140 -0
- package/docs/capture-warnings.md +57 -0
- package/docs/config.md +489 -0
- package/docs/deployment.md +289 -0
- package/docs/design-system.md +162 -0
- package/docs/github.md +107 -0
- package/docs/hosted.md +764 -0
- package/docs/images/board-overview.png +0 -0
- package/docs/images/journeys.png +0 -0
- package/docs/images/live-comment.png +0 -0
- package/docs/images/phone-comment.png +0 -0
- package/docs/images/phone-overview.png +0 -0
- package/docs/images/variant.png +0 -0
- package/docs/interaction-audits.md +40 -0
- package/docs/journeys.md +137 -0
- package/docs/local-accounts.md +80 -0
- package/docs/notifications.md +237 -0
- package/docs/page-checks.md +69 -0
- package/docs/performance.md +174 -0
- package/docs/starter-agent.md +325 -0
- package/docs/updates.md +87 -0
- package/examples/audits/interaction.audit.json +17 -0
- package/package.json +132 -4
- package/src/accounts/avatar.js +121 -0
- package/src/accounts/local.js +464 -0
- package/src/agent/acp/command.js +204 -0
- package/src/agent/acp/detect.js +75 -0
- package/src/agent/acp/sandbox.js +160 -0
- package/src/agent/agent-health.js +75 -0
- package/src/agent/backends/acp.js +1297 -0
- package/src/agent/backends/index.js +61 -0
- package/src/agent/backends/openai.js +308 -0
- package/src/agent/composer.js +221 -0
- package/src/agent/config.js +338 -0
- package/src/agent/context.js +191 -0
- package/src/agent/control.js +122 -0
- package/src/agent/errors.js +17 -0
- package/src/agent/feedback-loop.js +137 -0
- package/src/agent/manage.js +94 -0
- package/src/agent/next-steps.js +51 -0
- package/src/agent/owner-mcp.js +38 -0
- package/src/agent/plain-text.js +52 -0
- package/src/agent/ratings-core.js +213 -0
- package/src/agent/ratings.js +100 -0
- package/src/agent/reply-text.js +67 -0
- package/src/agent/routes.js +1065 -0
- package/src/agent/run.js +326 -0
- package/src/agent/settings-routes.js +704 -0
- package/src/agent/showable.js +23 -0
- package/src/agent/starter/approvals.js +181 -0
- package/src/agent/starter/build-image.js +90 -0
- package/src/agent/starter/cli.js +163 -0
- package/src/agent/starter/disk.js +53 -0
- package/src/agent/starter/doctor.js +180 -0
- package/src/agent/starter/egress-proxy.js +351 -0
- package/src/agent/starter/gate.js +90 -0
- package/src/agent/starter/loop.js +114 -0
- package/src/agent/starter/model-relay.js +289 -0
- package/src/agent/starter/pause.js +14 -0
- package/src/agent/starter/probe.js +272 -0
- package/src/agent/starter/prompt.js +41 -0
- package/src/agent/starter/push-guard.js +146 -0
- package/src/agent/starter/rpc.js +164 -0
- package/src/agent/starter/runtime.js +287 -0
- package/src/agent/starter/sandbox-main.js +121 -0
- package/src/agent/starter/settings.js +290 -0
- package/src/agent/starter/source.js +69 -0
- package/src/agent/starter/supervisor.js +755 -0
- package/src/agent/starter/sweep.js +46 -0
- package/src/agent/starter/tools.js +524 -0
- package/src/agent/starter/watch.js +84 -0
- package/src/agent/starter/worktree.js +409 -0
- package/src/agent/text-diff.js +210 -0
- package/src/agent/thread-history.js +223 -0
- package/src/agent/threads.js +399 -0
- package/src/agent/tools.js +200 -0
- package/src/agent/turn-changes.js +424 -0
- package/src/agent/undo-state.js +26 -0
- package/src/agent/user-settings.js +308 -0
- package/src/auth/record.js +375 -0
- package/src/broken-links.js +164 -0
- package/src/buzz/client.js +75 -0
- package/src/buzz/config.js +151 -0
- package/src/buzz/environment.js +13 -0
- package/src/buzz/gateway.js +192 -0
- package/src/buzz/routes.js +61 -0
- package/src/buzz/signer-child.js +205 -0
- package/src/buzz/store.js +89 -0
- package/src/buzz/transport.js +60 -0
- package/src/buzz/verify.js +140 -0
- package/src/capture/actions.js +159 -0
- package/src/capture/capture.js +776 -0
- package/src/capture/dom-names.js +189 -0
- package/src/capture/element-map.js +509 -0
- package/src/capture/live.js +827 -0
- package/src/capture/media-scheme.js +317 -0
- package/src/capture/network-summary.js +86 -0
- package/src/capture/pacing.js +34 -0
- package/src/capture/requests.js +167 -0
- package/src/capture/sanitize.js +752 -0
- package/src/checks/accessibility.js +134 -0
- package/src/checks/budgets.js +58 -0
- package/src/checks/cli.js +55 -0
- package/src/checks/comparison.js +113 -0
- package/src/checks/environments.js +32 -0
- package/src/checks/flow-config.js +82 -0
- package/src/checks/flow-origin.js +53 -0
- package/src/checks/flows.js +398 -0
- package/src/checks/lighthouse-auth.js +145 -0
- package/src/checks/lighthouse-findings.js +130 -0
- package/src/checks/lighthouse.js +395 -0
- package/src/checks/robots.js +85 -0
- package/src/checks/routes.js +52 -0
- package/src/checks/run-control.js +259 -0
- package/src/checks/runner.js +724 -0
- package/src/checks/schema.js +133 -0
- package/src/checks/seo-analysis.js +443 -0
- package/src/checks/seo-probe.js +190 -0
- package/src/checks/seo-snapshot.js +99 -0
- package/src/checks/store.js +214 -0
- package/src/checks/transport.js +47 -0
- package/src/cli/args.js +319 -0
- package/src/cli/capture.js +79 -0
- package/src/cli/commands.js +92 -0
- package/src/cli/comments.js +84 -0
- package/src/cli/help.js +243 -0
- package/src/cli/init.js +214 -0
- package/src/cli/misc.js +64 -0
- package/src/cli/notify.js +87 -0
- package/src/cli/open.js +22 -0
- package/src/cli/serve.js +274 -0
- package/src/cli/variant.js +82 -0
- package/src/cli/watch.js +70 -0
- package/src/cli.js +130 -0
- package/src/comment-fields.js +76 -0
- package/src/comments-cli.js +208 -0
- package/src/config-reload.js +100 -0
- package/src/config.js +1061 -0
- package/src/devices-cli.js +21 -0
- package/src/diff.js +95 -0
- package/src/discover.js +284 -0
- package/src/doctor.js +295 -0
- package/src/event-log.js +268 -0
- package/src/events-cli.js +102 -0
- package/src/events.js +113 -0
- package/src/export.js +119 -0
- package/src/feedback.js +246 -0
- package/src/github.js +551 -0
- package/src/hosted/LICENSE.md +105 -0
- package/src/hosted/access-cli.js +78 -0
- package/src/hosted/admin.js +682 -0
- package/src/hosted/agent-approvals.js +163 -0
- package/src/hosted/agent-policy.js +189 -0
- package/src/hosted/app.js +510 -0
- package/src/hosted/auth-routes.js +540 -0
- package/src/hosted/auth.js +536 -0
- package/src/hosted/backup.js +483 -0
- package/src/hosted/board-context.js +35 -0
- package/src/hosted/board-routes.js +486 -0
- package/src/hosted/board-view.js +94 -0
- package/src/hosted/clean-worker.js +24 -0
- package/src/hosted/cleaner.js +89 -0
- package/src/hosted/comments.js +355 -0
- package/src/hosted/console-routes.js +401 -0
- package/src/hosted/cookies.js +18 -0
- package/src/hosted/crypto.js +80 -0
- package/src/hosted/db.js +496 -0
- package/src/hosted/device-routes.js +175 -0
- package/src/hosted/device.js +233 -0
- package/src/hosted/events.js +63 -0
- package/src/hosted/guards.js +172 -0
- package/src/hosted/guest-view.js +65 -0
- package/src/hosted/http.js +39 -0
- package/src/hosted/import.js +300 -0
- package/src/hosted/mail.js +73 -0
- package/src/hosted/mcp-http.js +71 -0
- package/src/hosted/membership.js +114 -0
- package/src/hosted/operator.js +478 -0
- package/src/hosted/ops-cli.js +345 -0
- package/src/hosted/pages.js +146 -0
- package/src/hosted/permissions.js +113 -0
- package/src/hosted/presence.js +372 -0
- package/src/hosted/projects.js +124 -0
- package/src/hosted/public.js +109 -0
- package/src/hosted/push.js +622 -0
- package/src/hosted/quotas.js +115 -0
- package/src/hosted/ratelimit.js +96 -0
- package/src/hosted/rebind.js +316 -0
- package/src/hosted/retention.js +128 -0
- package/src/hosted/server.js +260 -0
- package/src/hosted/setup-link.js +64 -0
- package/src/hosted/setup.js +323 -0
- package/src/hosted/shares.js +444 -0
- package/src/hosted/tailscale.js +204 -0
- package/src/hosted/tenancy.js +301 -0
- package/src/hosted/token-api.js +455 -0
- package/src/hosted/token-events.js +60 -0
- package/src/hosted/tokens.js +57 -0
- package/src/hosted/turnstile.js +49 -0
- package/src/hosted/usercontent.js +154 -0
- package/src/hosted/zip.js +155 -0
- package/src/invocation.js +58 -0
- package/src/journey-paths.js +282 -0
- package/src/journeys.js +189 -0
- package/src/links.js +106 -0
- package/src/local-projects.js +182 -0
- package/src/lock.js +193 -0
- package/src/mcp.js +417 -0
- package/src/mentions.js +151 -0
- package/src/net.js +15 -0
- package/src/notify/config.js +275 -0
- package/src/notify/index.js +301 -0
- package/src/notify/lease.js +73 -0
- package/src/notify/messages.js +212 -0
- package/src/notify/run.js +174 -0
- package/src/pixeldiff.js +49 -0
- package/src/png.js +143 -0
- package/src/reactions.js +111 -0
- package/src/remote/cli.js +133 -0
- package/src/remote/credentials.js +62 -0
- package/src/remote/ids.js +12 -0
- package/src/remote/login.js +155 -0
- package/src/remote/remote-mcp.js +237 -0
- package/src/remote/remote-watch.js +291 -0
- package/src/remote/remote.js +313 -0
- package/src/remote/share-state.js +78 -0
- package/src/remote/share.js +369 -0
- package/src/remote/terminal.js +13 -0
- package/src/serve-state.js +64 -0
- package/src/server/app.js +824 -0
- package/src/server/devices.js +186 -0
- package/src/server/local-console.js +28 -0
- package/src/server/local-user.js +59 -0
- package/src/server/pairing-routes.js +159 -0
- package/src/server/share-routes.js +89 -0
- package/src/server/tailscale.js +97 -0
- package/src/sitemap.js +431 -0
- package/src/store.js +983 -0
- package/src/tokens.js +11 -0
- package/src/update/routes.js +245 -0
- package/src/watch.js +403 -0
- package/src/which.js +58 -0
|
@@ -0,0 +1,483 @@
|
|
|
1
|
+
// `shapewall hosted backup` and `restore`: a consistent copy of a hosted
|
|
2
|
+
// server's data folder, and putting one back.
|
|
3
|
+
//
|
|
4
|
+
// A backup is one folder, <to>/shapewall-backup-<UTC time>/ (0700):
|
|
5
|
+
// shapewall.db the database through VACUUM INTO: one consistent snapshot,
|
|
6
|
+
// taken while the server runs (WAL readers never block)
|
|
7
|
+
// files/ pushed captures, copied after the snapshot
|
|
8
|
+
// secret.key the signing key (0600), when the server made one
|
|
9
|
+
// quotas.json the operator's quotas, when there is one
|
|
10
|
+
// backup.json what it holds: schema version, row counts, the database's
|
|
11
|
+
// SHA-256, file count and bytes
|
|
12
|
+
// A SQLite write lock spans only the database snapshot and the listing of
|
|
13
|
+
// its board folders; the files are copied after it is released, so pushes and
|
|
14
|
+
// metadata writes keep working. A board folder that a later push swapped is
|
|
15
|
+
// copied again until it holds still, and files or folders that vanish while
|
|
16
|
+
// copying (retention, deletions) are skipped; backup.json lists both.
|
|
17
|
+
// Open pushes (staging/) are not backed up; they would expire anyway.
|
|
18
|
+
//
|
|
19
|
+
// A backup holds everything: accounts, passkey public keys, comments and
|
|
20
|
+
// captures, and the signing key. Keep it as private as the server itself.
|
|
21
|
+
//
|
|
22
|
+
// Restore copies a backup into a new, empty data folder, checks the database
|
|
23
|
+
// against backup.json (SHA-256 and PRAGMA integrity_check) and runs any newer
|
|
24
|
+
// migrations. `--check` is the restore drill: restore into a scratch folder
|
|
25
|
+
// next to the backup, start a server on it, check its health and its rows,
|
|
26
|
+
// then remove the scratch folder.
|
|
27
|
+
|
|
28
|
+
import fs from 'node:fs'
|
|
29
|
+
import path from 'node:path'
|
|
30
|
+
import http from 'node:http'
|
|
31
|
+
import { createHash, randomUUID } from 'node:crypto'
|
|
32
|
+
import { openDatabase, quietSqliteWarning } from './db.js'
|
|
33
|
+
import { ID_PATTERNS } from './crypto.js'
|
|
34
|
+
import { ID_RE } from '../config.js'
|
|
35
|
+
|
|
36
|
+
export const BACKUP_FORMAT = 'shapewall-hosted-backup'
|
|
37
|
+
const BACKUP_DIR_RE = /^shapewall-backup-\d{8}T\d{6}Z(?:-\d+)?$/
|
|
38
|
+
const COUNTED = ['users', 'passkeys', 'workspaces', 'memberships', 'projects', 'boards', 'comments', 'api_tokens', 'shares']
|
|
39
|
+
|
|
40
|
+
export class BackupError extends Error {}
|
|
41
|
+
|
|
42
|
+
const stampOf = (date) =>
|
|
43
|
+
date
|
|
44
|
+
.toISOString()
|
|
45
|
+
.replace(/[-:]/g, '')
|
|
46
|
+
.replace(/\.\d+Z$/, 'Z')
|
|
47
|
+
|
|
48
|
+
function sha256File(file) {
|
|
49
|
+
const hash = createHash('sha256')
|
|
50
|
+
const fd = fs.openSync(file, 'r')
|
|
51
|
+
const buffer = Buffer.alloc(1024 * 1024)
|
|
52
|
+
try {
|
|
53
|
+
let read
|
|
54
|
+
while ((read = fs.readSync(fd, buffer, 0, buffer.length, null)) > 0) hash.update(buffer.subarray(0, read))
|
|
55
|
+
} finally {
|
|
56
|
+
fs.closeSync(fd)
|
|
57
|
+
}
|
|
58
|
+
return hash.digest('hex')
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function openRaw(file, options = {}) {
|
|
62
|
+
quietSqliteWarning()
|
|
63
|
+
const { DatabaseSync } = await import('node:sqlite')
|
|
64
|
+
const conn = new DatabaseSync(file, options)
|
|
65
|
+
conn.exec('PRAGMA busy_timeout = 5000')
|
|
66
|
+
return conn
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Whole-database work on a copy (snapshot, counts, the drill): not a request
|
|
70
|
+
// path, so not through tenant() (src/hosted/tenancy.js).
|
|
71
|
+
function counts(conn) {
|
|
72
|
+
const out = {}
|
|
73
|
+
for (const table of COUNTED) out[table] = conn.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get().n
|
|
74
|
+
return out
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// Copies a folder tree; a folder that disappears while it is copied (a push
|
|
78
|
+
// commit swaps board folders) is tried again from the start.
|
|
79
|
+
function copyTree(from, to, stats, attempt = 0) {
|
|
80
|
+
const before = { ...stats }
|
|
81
|
+
let entries
|
|
82
|
+
try {
|
|
83
|
+
entries = fs.readdirSync(from, { withFileTypes: true })
|
|
84
|
+
} catch (error) {
|
|
85
|
+
if (error.code === 'ENOENT') return
|
|
86
|
+
throw error
|
|
87
|
+
}
|
|
88
|
+
fs.mkdirSync(to, { recursive: true, mode: 0o700 })
|
|
89
|
+
try {
|
|
90
|
+
for (const entry of entries) {
|
|
91
|
+
const source = path.join(from, entry.name)
|
|
92
|
+
const target = path.join(to, entry.name)
|
|
93
|
+
if (entry.isDirectory()) copyTree(source, target, stats)
|
|
94
|
+
else if (entry.isSymbolicLink()) throw new BackupError('backup files must not contain symbolic links')
|
|
95
|
+
else if (entry.isFile()) {
|
|
96
|
+
fs.copyFileSync(source, target, fs.constants.COPYFILE_FICLONE)
|
|
97
|
+
fs.chmodSync(target, 0o600)
|
|
98
|
+
stats.files += 1
|
|
99
|
+
stats.bytes += fs.statSync(target).size
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
} catch (error) {
|
|
103
|
+
if (error.code !== 'ENOENT' || attempt >= 3) throw error
|
|
104
|
+
fs.rmSync(to, { recursive: true, force: true })
|
|
105
|
+
Object.assign(stats, before)
|
|
106
|
+
copyTree(from, to, stats, attempt + 1)
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// The identity of a board folder: a push commit swaps in a new folder by
|
|
111
|
+
// rename, so a changed identity means the folder was replaced.
|
|
112
|
+
function folderId(dir) {
|
|
113
|
+
try {
|
|
114
|
+
const st = fs.statSync(dir, { bigint: true })
|
|
115
|
+
return `${st.dev}:${st.ino}:${st.birthtimeNs}`
|
|
116
|
+
} catch (error) {
|
|
117
|
+
if (error.code === 'ENOENT') return null
|
|
118
|
+
throw error
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const MISSING_RETRIES = 5
|
|
123
|
+
const MISSING_PAUSE_MS = 50
|
|
124
|
+
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
|
125
|
+
|
|
126
|
+
// Copies one board folder after the lock is released. Files that vanish while
|
|
127
|
+
// copying are skipped and listed; a folder swapped mid-copy is copied again.
|
|
128
|
+
// `locked` is the folder's identity read under the snapshot's lock: a push
|
|
129
|
+
// that committed after the snapshot but before this copy started shows up as
|
|
130
|
+
// a different identity. A push commit renames the old folder away before
|
|
131
|
+
// renaming the new one in, so a missing folder is looked for again a few
|
|
132
|
+
// times; it only counts as deleted once the live database has no row for it.
|
|
133
|
+
async function copyBoard(from, to, stats, rel, { locked, rowExists }) {
|
|
134
|
+
let changed = false
|
|
135
|
+
for (let attempt = 0, misses = 0; ;) {
|
|
136
|
+
const id = folderId(from)
|
|
137
|
+
if (!id) {
|
|
138
|
+
if (misses < MISSING_RETRIES) {
|
|
139
|
+
misses += 1
|
|
140
|
+
await pause(MISSING_PAUSE_MS)
|
|
141
|
+
continue
|
|
142
|
+
}
|
|
143
|
+
if (!rowExists()) return { vanished: true }
|
|
144
|
+
return { changed, missing: [`${rel}/`] }
|
|
145
|
+
}
|
|
146
|
+
if (id !== locked) changed = true
|
|
147
|
+
const local = { files: 0, bytes: 0, missing: [] }
|
|
148
|
+
fs.rmSync(to, { recursive: true, force: true })
|
|
149
|
+
copyLive(from, to, local, rel)
|
|
150
|
+
if (folderId(from) === id || attempt >= 3) {
|
|
151
|
+
stats.files += local.files
|
|
152
|
+
stats.bytes += local.bytes
|
|
153
|
+
return { changed: changed || attempt > 0, missing: local.missing }
|
|
154
|
+
}
|
|
155
|
+
attempt += 1
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function copyLive(from, to, stats, rel) {
|
|
160
|
+
let entries
|
|
161
|
+
try {
|
|
162
|
+
entries = fs.readdirSync(from, { withFileTypes: true })
|
|
163
|
+
} catch (error) {
|
|
164
|
+
if (error.code !== 'ENOENT') throw error
|
|
165
|
+
stats.missing.push(`${rel}/`)
|
|
166
|
+
return
|
|
167
|
+
}
|
|
168
|
+
fs.mkdirSync(to, { recursive: true, mode: 0o700 })
|
|
169
|
+
for (const entry of entries) {
|
|
170
|
+
const source = path.join(from, entry.name)
|
|
171
|
+
const target = path.join(to, entry.name)
|
|
172
|
+
if (entry.isDirectory()) copyLive(source, target, stats, `${rel}/${entry.name}`)
|
|
173
|
+
else if (entry.isSymbolicLink()) throw new BackupError('backup files must not contain symbolic links')
|
|
174
|
+
else if (entry.isFile()) {
|
|
175
|
+
try {
|
|
176
|
+
fs.copyFileSync(source, target, fs.constants.COPYFILE_FICLONE)
|
|
177
|
+
} catch (error) {
|
|
178
|
+
if (error.code !== 'ENOENT') throw error
|
|
179
|
+
stats.missing.push(`${rel}/${entry.name}`)
|
|
180
|
+
continue
|
|
181
|
+
}
|
|
182
|
+
fs.chmodSync(target, 0o600)
|
|
183
|
+
stats.files += 1
|
|
184
|
+
stats.bytes += fs.statSync(target).size
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function fileDigests(dir) {
|
|
190
|
+
const files = {}
|
|
191
|
+
const walk = (folder) => {
|
|
192
|
+
for (const entry of fs.readdirSync(folder, { withFileTypes: true })) {
|
|
193
|
+
const file = path.join(folder, entry.name)
|
|
194
|
+
if (entry.isSymbolicLink()) throw new BackupError('backup files must not contain symbolic links')
|
|
195
|
+
if (entry.isDirectory()) walk(file)
|
|
196
|
+
else if (entry.isFile() && file !== path.join(dir, 'backup.json'))
|
|
197
|
+
files[path.relative(dir, file).split(path.sep).join('/')] = sha256File(file)
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
walk(dir)
|
|
201
|
+
return files
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function copySmall(from, to) {
|
|
205
|
+
if (!fs.existsSync(from)) return false
|
|
206
|
+
fs.copyFileSync(from, to)
|
|
207
|
+
fs.chmodSync(to, 0o600)
|
|
208
|
+
return true
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export async function backup({ dataDir, to, now = new Date(), keepDays = null }) {
|
|
212
|
+
const source = path.resolve(dataDir)
|
|
213
|
+
const dbFile = path.join(source, 'shapewall.db')
|
|
214
|
+
if (!fs.existsSync(dbFile)) throw new BackupError(`no hosted database at ${dbFile} (is --data right?)`)
|
|
215
|
+
const root = path.resolve(to)
|
|
216
|
+
if (root === source || root.startsWith(source + path.sep)) throw new BackupError('back up to a folder outside the data folder')
|
|
217
|
+
fs.mkdirSync(root, { recursive: true, mode: 0o700 })
|
|
218
|
+
let name = `shapewall-backup-${stampOf(now)}`
|
|
219
|
+
for (let n = 2; fs.existsSync(path.join(root, name)); n += 1) name = `shapewall-backup-${stampOf(now)}-${n}`
|
|
220
|
+
const dir = path.join(root, name)
|
|
221
|
+
const partial = `${dir}.${randomUUID()}.partial`
|
|
222
|
+
fs.mkdirSync(partial, { mode: 0o700 })
|
|
223
|
+
try {
|
|
224
|
+
const conn = await openRaw(dbFile)
|
|
225
|
+
let schema
|
|
226
|
+
let rows
|
|
227
|
+
let boards
|
|
228
|
+
const stats = { files: 0, bytes: 0 }
|
|
229
|
+
try {
|
|
230
|
+
// Push commits, retention and deletions all take the same SQLite write
|
|
231
|
+
// lock. Hold it only for the snapshot and the board list; the files are
|
|
232
|
+
// copied after it is released so the server's writes do not wait.
|
|
233
|
+
conn.exec('BEGIN IMMEDIATE')
|
|
234
|
+
const reader = await openRaw(dbFile, { readOnly: true })
|
|
235
|
+
try {
|
|
236
|
+
reader.prepare('VACUUM INTO ?').run(path.join(partial, 'shapewall.db'))
|
|
237
|
+
} finally {
|
|
238
|
+
reader.close()
|
|
239
|
+
}
|
|
240
|
+
fs.chmodSync(path.join(partial, 'shapewall.db'), 0o600)
|
|
241
|
+
const copy = await openRaw(path.join(partial, 'shapewall.db'), { readOnly: true })
|
|
242
|
+
try {
|
|
243
|
+
schema = copy.prepare('PRAGMA user_version').get().user_version
|
|
244
|
+
rows = counts(copy)
|
|
245
|
+
boards = copy.prepare('SELECT workspace_id, project_id, board_id FROM boards ORDER BY workspace_id, project_id, board_id').all()
|
|
246
|
+
} finally {
|
|
247
|
+
copy.close()
|
|
248
|
+
}
|
|
249
|
+
// Each board folder's identity while the lock is still held, so a push
|
|
250
|
+
// that commits before its folder is copied is noticed.
|
|
251
|
+
for (const row of boards) {
|
|
252
|
+
if (!ID_PATTERNS.workspace.test(row.workspace_id) || !ID_PATTERNS.project.test(row.project_id) || !ID_RE.test(row.board_id))
|
|
253
|
+
continue
|
|
254
|
+
row.folderId = folderId(path.join(source, 'files', row.workspace_id, row.project_id, row.board_id))
|
|
255
|
+
}
|
|
256
|
+
conn.exec('COMMIT')
|
|
257
|
+
} finally {
|
|
258
|
+
conn.close()
|
|
259
|
+
}
|
|
260
|
+
const vanished = []
|
|
261
|
+
const changed = []
|
|
262
|
+
const missing = []
|
|
263
|
+
// The live database, read only: is a board whose folder is gone deleted?
|
|
264
|
+
const live = await openRaw(dbFile, { readOnly: true })
|
|
265
|
+
try {
|
|
266
|
+
const boardRow = live.prepare('SELECT 1 FROM boards WHERE workspace_id = ? AND project_id = ? AND board_id = ?')
|
|
267
|
+
for (const row of boards) {
|
|
268
|
+
const rel = `${row.workspace_id}/${row.project_id}/${row.board_id}`
|
|
269
|
+
if (!ID_PATTERNS.workspace.test(row.workspace_id) || !ID_PATTERNS.project.test(row.project_id) || !ID_RE.test(row.board_id))
|
|
270
|
+
continue
|
|
271
|
+
const result = await copyBoard(path.join(source, 'files', rel), path.join(partial, 'files', rel), stats, rel, {
|
|
272
|
+
locked: row.folderId,
|
|
273
|
+
rowExists: () => Boolean(boardRow.get(row.workspace_id, row.project_id, row.board_id)),
|
|
274
|
+
})
|
|
275
|
+
if (result.vanished) vanished.push(rel)
|
|
276
|
+
else {
|
|
277
|
+
if (result.changed) changed.push(rel)
|
|
278
|
+
missing.push(...result.missing)
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
} finally {
|
|
282
|
+
live.close()
|
|
283
|
+
}
|
|
284
|
+
if (vanished.length || changed.length || missing.length) stats.skipped = { vanished, changed, missing }
|
|
285
|
+
const { loadCredentials } = await import('./server.js')
|
|
286
|
+
loadCredentials()
|
|
287
|
+
let secret
|
|
288
|
+
if (process.env.SHAPEWALL_HOSTED_SECRET) {
|
|
289
|
+
fs.writeFileSync(path.join(partial, 'secret.key'), `${process.env.SHAPEWALL_HOSTED_SECRET}\n`, { mode: 0o600 })
|
|
290
|
+
secret = true
|
|
291
|
+
} else secret = copySmall(path.join(source, 'secret.key'), path.join(partial, 'secret.key'))
|
|
292
|
+
const quotas = copySmall(path.join(source, 'quotas.json'), path.join(partial, 'quotas.json'))
|
|
293
|
+
const manifest = {
|
|
294
|
+
format: BACKUP_FORMAT,
|
|
295
|
+
version: 1,
|
|
296
|
+
createdAt: now.toISOString(),
|
|
297
|
+
source,
|
|
298
|
+
schema,
|
|
299
|
+
counts: rows,
|
|
300
|
+
db: {
|
|
301
|
+
file: 'shapewall.db',
|
|
302
|
+
bytes: fs.statSync(path.join(partial, 'shapewall.db')).size,
|
|
303
|
+
sha256: sha256File(path.join(partial, 'shapewall.db')),
|
|
304
|
+
},
|
|
305
|
+
files: stats,
|
|
306
|
+
hashes: fileDigests(partial),
|
|
307
|
+
secret,
|
|
308
|
+
quotas,
|
|
309
|
+
}
|
|
310
|
+
fs.writeFileSync(path.join(partial, 'backup.json'), `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 })
|
|
311
|
+
fs.renameSync(partial, dir)
|
|
312
|
+
const pruned = keepDays ? pruneBackups(root, { keepDays, now, keep: dir }) : []
|
|
313
|
+
return { dir, manifest, pruned }
|
|
314
|
+
} catch (error) {
|
|
315
|
+
fs.rmSync(partial, { recursive: true, force: true })
|
|
316
|
+
throw error
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
function readManifest(dir) {
|
|
321
|
+
try {
|
|
322
|
+
const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'backup.json'), 'utf8'))
|
|
323
|
+
return manifest?.format === BACKUP_FORMAT ? manifest : null
|
|
324
|
+
} catch {
|
|
325
|
+
return null
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
// Backups older than keepDays in `root`. Only folders this tool wrote (the
|
|
330
|
+
// name pattern and a readable backup.json) are ever removed.
|
|
331
|
+
export function pruneBackups(root, { keepDays, now = new Date(), keep = null }) {
|
|
332
|
+
const cutoff = now.getTime() - keepDays * 86_400_000
|
|
333
|
+
const removed = []
|
|
334
|
+
for (const name of fs.readdirSync(root)) {
|
|
335
|
+
if (!BACKUP_DIR_RE.test(name)) continue
|
|
336
|
+
const dir = path.join(root, name)
|
|
337
|
+
if (dir === keep) continue
|
|
338
|
+
const manifest = readManifest(dir)
|
|
339
|
+
const created = Date.parse(manifest?.createdAt || '')
|
|
340
|
+
if (!manifest || !Number.isFinite(created) || created >= cutoff) continue
|
|
341
|
+
fs.rmSync(dir, { recursive: true, force: true })
|
|
342
|
+
removed.push(dir)
|
|
343
|
+
}
|
|
344
|
+
return removed
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// A backup folder, or a folder of backups (the newest one is used).
|
|
348
|
+
export function findBackup(from) {
|
|
349
|
+
const dir = path.resolve(from)
|
|
350
|
+
if (readManifest(dir)) return dir
|
|
351
|
+
let names = []
|
|
352
|
+
try {
|
|
353
|
+
names = fs.readdirSync(dir).filter((name) => BACKUP_DIR_RE.test(name) && readManifest(path.join(dir, name)))
|
|
354
|
+
} catch {
|
|
355
|
+
throw new BackupError(`no backup at ${dir}`)
|
|
356
|
+
}
|
|
357
|
+
if (!names.length) throw new BackupError(`no backup in ${dir} (looked for shapewall-backup-*/backup.json)`)
|
|
358
|
+
return path.join(dir, names.sort().at(-1))
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
export async function restore({ from, data }) {
|
|
362
|
+
const dir = findBackup(from)
|
|
363
|
+
const manifest = readManifest(dir)
|
|
364
|
+
const target = path.resolve(data)
|
|
365
|
+
if (target === dir || target.startsWith(dir + path.sep)) throw new BackupError('restore outside the backup folder')
|
|
366
|
+
if (fs.existsSync(target) && fs.readdirSync(target).length)
|
|
367
|
+
throw new BackupError(`${target} is not empty: restore into a new folder, then point the server at it`)
|
|
368
|
+
if (manifest.db?.file !== 'shapewall.db' || !manifest.files || manifest.version !== 1)
|
|
369
|
+
throw new BackupError('unsupported backup manifest')
|
|
370
|
+
if (manifest.hashes) {
|
|
371
|
+
const actual = fileDigests(dir)
|
|
372
|
+
if (JSON.stringify(Object.entries(actual).sort()) !== JSON.stringify(Object.entries(manifest.hashes).sort()))
|
|
373
|
+
throw new BackupError('backup files do not match their checksums (damaged copy)')
|
|
374
|
+
}
|
|
375
|
+
const dbCopy = path.join(dir, 'shapewall.db')
|
|
376
|
+
if (sha256File(dbCopy) !== manifest.db.sha256)
|
|
377
|
+
throw new BackupError('the database in this backup does not match its backup.json (damaged copy)')
|
|
378
|
+
fs.mkdirSync(target, { recursive: true, mode: 0o700 })
|
|
379
|
+
fs.chmodSync(target, 0o700)
|
|
380
|
+
try {
|
|
381
|
+
fs.copyFileSync(dbCopy, path.join(target, 'shapewall.db'))
|
|
382
|
+
fs.chmodSync(path.join(target, 'shapewall.db'), 0o600)
|
|
383
|
+
const check = await openRaw(path.join(target, 'shapewall.db'))
|
|
384
|
+
let integrity
|
|
385
|
+
try {
|
|
386
|
+
integrity = check.prepare('PRAGMA integrity_check').get()
|
|
387
|
+
} finally {
|
|
388
|
+
check.close()
|
|
389
|
+
}
|
|
390
|
+
if (Object.values(integrity)[0] !== 'ok') throw new BackupError(`integrity check failed: ${Object.values(integrity)[0]}`)
|
|
391
|
+
const stats = { files: 0, bytes: 0 }
|
|
392
|
+
copyTree(path.join(dir, 'files'), path.join(target, 'files'), stats)
|
|
393
|
+
if (stats.files !== manifest.files.files)
|
|
394
|
+
throw new BackupError(`expected ${manifest.files.files} files, copied ${stats.files} (damaged copy?)`)
|
|
395
|
+
copySmall(path.join(dir, 'secret.key'), path.join(target, 'secret.key'))
|
|
396
|
+
copySmall(path.join(dir, 'quotas.json'), path.join(target, 'quotas.json'))
|
|
397
|
+
// Newer migrations, when this build is newer than the backup.
|
|
398
|
+
const conn = await openDatabase(path.join(target, 'shapewall.db'))
|
|
399
|
+
const schema = conn.prepare('PRAGMA user_version').get().user_version
|
|
400
|
+
const rows = counts(conn)
|
|
401
|
+
conn.close()
|
|
402
|
+
return { from: dir, data: target, manifest, schema, counts: rows, files: stats }
|
|
403
|
+
} catch (error) {
|
|
404
|
+
fs.rmSync(target, { recursive: true, force: true })
|
|
405
|
+
throw error
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
// The restore drill: restore into a scratch folder beside the backup (on the
|
|
410
|
+
// same disk, not a RAM /tmp), start a server on a free loopback port, check
|
|
411
|
+
// /api/health, the row counts and that every board's folder is there, stop
|
|
412
|
+
// it and remove the scratch folder. Returns { ok, checks: [...] }.
|
|
413
|
+
export async function restoreDrill({ from, scratch = null, out = () => {} }) {
|
|
414
|
+
const dir = findBackup(from)
|
|
415
|
+
const scratchDir = path.resolve(scratch || path.join(path.dirname(dir), `.restore-drill-${process.pid}-${Date.now()}`))
|
|
416
|
+
if (fs.existsSync(scratchDir)) throw new BackupError('restore drill scratch folder already exists; choose a new folder')
|
|
417
|
+
const checks = []
|
|
418
|
+
const check = (name, ok, detail = '') => {
|
|
419
|
+
checks.push({ name, ok, detail })
|
|
420
|
+
out(`${ok ? 'ok ' : 'FAIL'} ${name}${detail ? `: ${detail}` : ''}`)
|
|
421
|
+
}
|
|
422
|
+
let server = null
|
|
423
|
+
let app = null
|
|
424
|
+
let conn = null
|
|
425
|
+
try {
|
|
426
|
+
const restored = await restore({ from: dir, data: scratchDir })
|
|
427
|
+
check('restore', true, `${restored.files.files} files, schema ${restored.schema}`)
|
|
428
|
+
for (const table of COUNTED) {
|
|
429
|
+
const expected = restored.manifest.counts?.[table]
|
|
430
|
+
if (expected === undefined) continue
|
|
431
|
+
check(`rows in ${table}`, restored.counts[table] === expected, `${restored.counts[table]} (backup says ${expected})`)
|
|
432
|
+
}
|
|
433
|
+
conn = await openDatabase(path.join(scratchDir, 'shapewall.db'))
|
|
434
|
+
const boards = conn.prepare('SELECT workspace_id, project_id, board_id FROM boards').all()
|
|
435
|
+
// Boards deleted after the snapshot, while files were copied, are listed.
|
|
436
|
+
const vanished = new Set(restored.manifest.files.skipped?.vanished || [])
|
|
437
|
+
const missing = boards.filter(
|
|
438
|
+
(row) =>
|
|
439
|
+
!vanished.has(`${row.workspace_id}/${row.project_id}/${row.board_id}`) &&
|
|
440
|
+
!fs.existsSync(path.join(scratchDir, 'files', row.workspace_id, row.project_id, row.board_id)),
|
|
441
|
+
)
|
|
442
|
+
check(
|
|
443
|
+
'board folders',
|
|
444
|
+
!missing.length,
|
|
445
|
+
missing.length
|
|
446
|
+
? `${missing.length} of ${boards.length} missing`
|
|
447
|
+
: `${boards.length - vanished.size} present${vanished.size ? `, ${vanished.size} deleted while backing up` : ''}`,
|
|
448
|
+
)
|
|
449
|
+
const { createHostedApp } = await import('./app.js')
|
|
450
|
+
const { readSecret } = await import('./server.js')
|
|
451
|
+
server = http.createServer((req, res) => app(req, res))
|
|
452
|
+
await new Promise((resolve, reject) => {
|
|
453
|
+
server.once('error', reject)
|
|
454
|
+
server.listen(0, '127.0.0.1', resolve)
|
|
455
|
+
})
|
|
456
|
+
const port = server.address().port
|
|
457
|
+
app = createHostedApp({
|
|
458
|
+
db: conn,
|
|
459
|
+
dataDir: scratchDir,
|
|
460
|
+
origin: `http://localhost:${port}`,
|
|
461
|
+
secret: readSecret(scratchDir),
|
|
462
|
+
sweepEveryMs: 0,
|
|
463
|
+
})
|
|
464
|
+
const health = await new Promise((resolve) => {
|
|
465
|
+
http
|
|
466
|
+
.get({ host: '127.0.0.1', port, path: '/api/health' }, (res) => {
|
|
467
|
+
let body = ''
|
|
468
|
+
res.on('data', (chunk) => (body += chunk))
|
|
469
|
+
res.on('end', () => resolve({ status: res.statusCode, body }))
|
|
470
|
+
})
|
|
471
|
+
.on('error', (error) => resolve({ status: 0, body: error.message }))
|
|
472
|
+
})
|
|
473
|
+
check('server starts and answers /api/health', health.status === 200 && /"ok":true/.test(health.body), `HTTP ${health.status}`)
|
|
474
|
+
} catch (error) {
|
|
475
|
+
check('restore drill', false, error.message)
|
|
476
|
+
} finally {
|
|
477
|
+
if (server) await new Promise((resolve) => server.close(resolve))
|
|
478
|
+
if (app) await app.locals.close()
|
|
479
|
+
conn?.close()
|
|
480
|
+
fs.rmSync(scratchDir, { recursive: true, force: true })
|
|
481
|
+
}
|
|
482
|
+
return { ok: checks.every((item) => item.ok), backup: dir, checks }
|
|
483
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// Boards as the board API and the token API load them (src/hosted/app.js):
|
|
2
|
+
// a board's row, config and comments; the view a hosted board shows; and
|
|
3
|
+
// its images.
|
|
4
|
+
|
|
5
|
+
import { boardView } from '../server/app.js'
|
|
6
|
+
import { publicBoardView } from './board-view.js'
|
|
7
|
+
import { createCommentStore } from './comments.js'
|
|
8
|
+
import { HttpError } from './http.js'
|
|
9
|
+
import { boardConfig, createProjectStore } from './projects.js'
|
|
10
|
+
|
|
11
|
+
export function createBoardContext({ db, dataDir }) {
|
|
12
|
+
function boardContext(t, project, boardId) {
|
|
13
|
+
const row = createProjectStore(t, { db, dataDir }).board(project.id, boardId)
|
|
14
|
+
if (!row) throw new HttpError(404, 'unknown_board', 'No such board.')
|
|
15
|
+
const config = boardConfig(dataDir, t.workspaceId, project.id, row)
|
|
16
|
+
return { row, config, board: config.boards[0], comments: createCommentStore(t, project.id, row.board_id) }
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
return boardContext
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function hostedBoardView(ctx) {
|
|
23
|
+
const view = publicBoardView(boardView(ctx.config, ctx.board), ctx.board)
|
|
24
|
+
view.commentCounts = ctx.comments.counts()
|
|
25
|
+
view.capture = { running: false, state: 'idle', startedAt: null, finishedAt: null, done: null, total: null, failed: null, error: null }
|
|
26
|
+
view.pushedAt = ctx.row.pushed_at
|
|
27
|
+
return view
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function sendPng(res, file, cacheable) {
|
|
31
|
+
res.setHeader('Cache-Control', cacheable ? 'private, max-age=604800, immutable' : 'private, no-cache')
|
|
32
|
+
res.setHeader('Cross-Origin-Resource-Policy', 'same-origin')
|
|
33
|
+
res.setHeader('Content-Security-Policy', "default-src 'none'; sandbox")
|
|
34
|
+
res.type('png').sendFile(file, { dotfiles: 'allow' })
|
|
35
|
+
}
|