shapewall 0.0.0-stage → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (304) hide show
  1. package/CHANGELOG.md +139 -0
  2. package/Dockerfile.agent +45 -0
  3. package/LICENSE +202 -0
  4. package/NOTICE +12 -0
  5. package/README.md +651 -2
  6. package/THIRD_PARTY.md +165 -0
  7. package/bin/shapewall.js +29 -0
  8. package/dist/agent/package-lock.json +8384 -0
  9. package/dist/client/assets/BoardChat-D5SFd8HE.js +28 -0
  10. package/dist/client/assets/BuzzDialog-HWXBy52W.js +1 -0
  11. package/dist/client/assets/ChecksPanel-DTdImWoD.js +1 -0
  12. package/dist/client/assets/History-CEv6b-av.js +1 -0
  13. package/dist/client/assets/LocalSettings-C4vriEVQ.js +1 -0
  14. package/dist/client/assets/Management-DCfhcEUY.js +1 -0
  15. package/dist/client/assets/QuickConnect-DQGgUsl1.js +1 -0
  16. package/dist/client/assets/ShareDialog-C_plIto0.js +2 -0
  17. package/dist/client/assets/app-update-dialog-2hPz_N9G.js +1 -0
  18. package/dist/client/assets/circle-x-BZMPi8Zf.js +1 -0
  19. package/dist/client/assets/index-ByQRKVit.css +2 -0
  20. package/dist/client/assets/index-DweQ8Qtu.js +29 -0
  21. package/dist/client/assets/refresh-cw-CRow0Nbq.js +1 -0
  22. package/dist/client/assets/runtime-Bft6x_6E.js +4 -0
  23. package/dist/client/assets/square-A8EnIYLR.js +1 -0
  24. package/dist/client/assets/tts-dist-DYlBQmwG.js +18 -0
  25. package/dist/client/assets/tts-en-g2p-BPUtBFGG.js +1 -0
  26. package/dist/client/assets/tts-ort-wasm-simd-threaded-qh98TCA_.wasm +0 -0
  27. package/dist/client/assets/tts-ort.wasm.bundle.min-rEkcs_ow.js +12 -0
  28. package/dist/client/assets/tts-rolldown-runtime-D8NudN84.js +1 -0
  29. package/dist/client/assets/tts-spacy-en-tokenizer-6SNMl--A.json +1 -0
  30. package/dist/client/assets/tts-tts.worker-Bzm6aWbv.js +1 -0
  31. package/dist/client/assets/tts-us_gold-DcjeNMH1.json +92589 -0
  32. package/dist/client/assets/tts-us_silver-C964mEB5.json +93363 -0
  33. package/dist/client/assets/ui-CLbws0rx.js +41 -0
  34. package/dist/client/build-info.json +1 -0
  35. package/dist/client/favicon.svg +7 -0
  36. package/dist/client/icons/apple-touch-icon.png +0 -0
  37. package/dist/client/icons/icon-192.png +0 -0
  38. package/dist/client/icons/icon-512.png +0 -0
  39. package/dist/client/icons/maskable-512.png +0 -0
  40. package/dist/client/index.html +25 -0
  41. package/dist/client/manifest.webmanifest +15 -0
  42. package/dist/client/sw.js +53 -0
  43. package/dist/client/theme-boot.js +27 -0
  44. package/dist/console/assets/LocalAccountsApp-BQCcmj3x.js +1 -0
  45. package/dist/console/assets/LocalConsole-Bb2FUDs7.js +1 -0
  46. package/dist/console/assets/app-update-dialog-DIeYepLZ.js +1 -0
  47. package/dist/console/assets/console-C7uJLLOn.js +51 -0
  48. package/dist/console/assets/console-R8IBY3Am.css +2 -0
  49. package/dist/console/build-info.json +1 -0
  50. package/dist/console/console.html +21 -0
  51. package/dist/console/favicon.svg +7 -0
  52. package/dist/console/icons/apple-touch-icon.png +0 -0
  53. package/dist/console/icons/icon-192.png +0 -0
  54. package/dist/console/icons/icon-512.png +0 -0
  55. package/dist/console/icons/maskable-512.png +0 -0
  56. package/dist/console/manifest.webmanifest +15 -0
  57. package/dist/console/theme-boot.js +27 -0
  58. package/docs/agent-hardening.md +342 -0
  59. package/docs/agent-recipes.md +187 -0
  60. package/docs/agent-workflow.md +320 -0
  61. package/docs/agents.md +894 -0
  62. package/docs/architecture.md +242 -0
  63. package/docs/buzz.md +140 -0
  64. package/docs/capture-warnings.md +57 -0
  65. package/docs/config.md +489 -0
  66. package/docs/deployment.md +289 -0
  67. package/docs/design-system.md +162 -0
  68. package/docs/github.md +107 -0
  69. package/docs/hosted.md +764 -0
  70. package/docs/images/board-overview.png +0 -0
  71. package/docs/images/journeys.png +0 -0
  72. package/docs/images/live-comment.png +0 -0
  73. package/docs/images/phone-comment.png +0 -0
  74. package/docs/images/phone-overview.png +0 -0
  75. package/docs/images/variant.png +0 -0
  76. package/docs/interaction-audits.md +40 -0
  77. package/docs/journeys.md +137 -0
  78. package/docs/local-accounts.md +80 -0
  79. package/docs/notifications.md +237 -0
  80. package/docs/page-checks.md +69 -0
  81. package/docs/performance.md +174 -0
  82. package/docs/starter-agent.md +325 -0
  83. package/docs/updates.md +87 -0
  84. package/examples/audits/interaction.audit.json +17 -0
  85. package/package.json +132 -4
  86. package/src/accounts/avatar.js +121 -0
  87. package/src/accounts/local.js +464 -0
  88. package/src/agent/acp/command.js +204 -0
  89. package/src/agent/acp/detect.js +75 -0
  90. package/src/agent/acp/sandbox.js +160 -0
  91. package/src/agent/agent-health.js +75 -0
  92. package/src/agent/backends/acp.js +1297 -0
  93. package/src/agent/backends/index.js +61 -0
  94. package/src/agent/backends/openai.js +308 -0
  95. package/src/agent/composer.js +221 -0
  96. package/src/agent/config.js +338 -0
  97. package/src/agent/context.js +191 -0
  98. package/src/agent/control.js +122 -0
  99. package/src/agent/errors.js +17 -0
  100. package/src/agent/feedback-loop.js +137 -0
  101. package/src/agent/manage.js +94 -0
  102. package/src/agent/next-steps.js +51 -0
  103. package/src/agent/owner-mcp.js +38 -0
  104. package/src/agent/plain-text.js +52 -0
  105. package/src/agent/ratings-core.js +213 -0
  106. package/src/agent/ratings.js +100 -0
  107. package/src/agent/reply-text.js +67 -0
  108. package/src/agent/routes.js +1065 -0
  109. package/src/agent/run.js +326 -0
  110. package/src/agent/settings-routes.js +704 -0
  111. package/src/agent/showable.js +23 -0
  112. package/src/agent/starter/approvals.js +181 -0
  113. package/src/agent/starter/build-image.js +90 -0
  114. package/src/agent/starter/cli.js +163 -0
  115. package/src/agent/starter/disk.js +53 -0
  116. package/src/agent/starter/doctor.js +180 -0
  117. package/src/agent/starter/egress-proxy.js +351 -0
  118. package/src/agent/starter/gate.js +90 -0
  119. package/src/agent/starter/loop.js +114 -0
  120. package/src/agent/starter/model-relay.js +289 -0
  121. package/src/agent/starter/pause.js +14 -0
  122. package/src/agent/starter/probe.js +272 -0
  123. package/src/agent/starter/prompt.js +41 -0
  124. package/src/agent/starter/push-guard.js +146 -0
  125. package/src/agent/starter/rpc.js +164 -0
  126. package/src/agent/starter/runtime.js +287 -0
  127. package/src/agent/starter/sandbox-main.js +121 -0
  128. package/src/agent/starter/settings.js +290 -0
  129. package/src/agent/starter/source.js +69 -0
  130. package/src/agent/starter/supervisor.js +755 -0
  131. package/src/agent/starter/sweep.js +46 -0
  132. package/src/agent/starter/tools.js +524 -0
  133. package/src/agent/starter/watch.js +84 -0
  134. package/src/agent/starter/worktree.js +409 -0
  135. package/src/agent/text-diff.js +210 -0
  136. package/src/agent/thread-history.js +223 -0
  137. package/src/agent/threads.js +399 -0
  138. package/src/agent/tools.js +200 -0
  139. package/src/agent/turn-changes.js +424 -0
  140. package/src/agent/undo-state.js +26 -0
  141. package/src/agent/user-settings.js +308 -0
  142. package/src/auth/record.js +375 -0
  143. package/src/broken-links.js +164 -0
  144. package/src/buzz/client.js +75 -0
  145. package/src/buzz/config.js +151 -0
  146. package/src/buzz/environment.js +13 -0
  147. package/src/buzz/gateway.js +192 -0
  148. package/src/buzz/routes.js +61 -0
  149. package/src/buzz/signer-child.js +205 -0
  150. package/src/buzz/store.js +89 -0
  151. package/src/buzz/transport.js +60 -0
  152. package/src/buzz/verify.js +140 -0
  153. package/src/capture/actions.js +159 -0
  154. package/src/capture/capture.js +776 -0
  155. package/src/capture/dom-names.js +189 -0
  156. package/src/capture/element-map.js +509 -0
  157. package/src/capture/live.js +827 -0
  158. package/src/capture/media-scheme.js +317 -0
  159. package/src/capture/network-summary.js +86 -0
  160. package/src/capture/pacing.js +34 -0
  161. package/src/capture/requests.js +167 -0
  162. package/src/capture/sanitize.js +752 -0
  163. package/src/checks/accessibility.js +134 -0
  164. package/src/checks/budgets.js +58 -0
  165. package/src/checks/cli.js +55 -0
  166. package/src/checks/comparison.js +113 -0
  167. package/src/checks/environments.js +32 -0
  168. package/src/checks/flow-config.js +82 -0
  169. package/src/checks/flow-origin.js +53 -0
  170. package/src/checks/flows.js +398 -0
  171. package/src/checks/lighthouse-auth.js +145 -0
  172. package/src/checks/lighthouse-findings.js +130 -0
  173. package/src/checks/lighthouse.js +395 -0
  174. package/src/checks/robots.js +85 -0
  175. package/src/checks/routes.js +52 -0
  176. package/src/checks/run-control.js +259 -0
  177. package/src/checks/runner.js +724 -0
  178. package/src/checks/schema.js +133 -0
  179. package/src/checks/seo-analysis.js +443 -0
  180. package/src/checks/seo-probe.js +190 -0
  181. package/src/checks/seo-snapshot.js +99 -0
  182. package/src/checks/store.js +214 -0
  183. package/src/checks/transport.js +47 -0
  184. package/src/cli/args.js +319 -0
  185. package/src/cli/capture.js +79 -0
  186. package/src/cli/commands.js +92 -0
  187. package/src/cli/comments.js +84 -0
  188. package/src/cli/help.js +243 -0
  189. package/src/cli/init.js +214 -0
  190. package/src/cli/misc.js +64 -0
  191. package/src/cli/notify.js +87 -0
  192. package/src/cli/open.js +22 -0
  193. package/src/cli/serve.js +274 -0
  194. package/src/cli/variant.js +82 -0
  195. package/src/cli/watch.js +70 -0
  196. package/src/cli.js +130 -0
  197. package/src/comment-fields.js +76 -0
  198. package/src/comments-cli.js +208 -0
  199. package/src/config-reload.js +100 -0
  200. package/src/config.js +1061 -0
  201. package/src/devices-cli.js +21 -0
  202. package/src/diff.js +95 -0
  203. package/src/discover.js +284 -0
  204. package/src/doctor.js +295 -0
  205. package/src/event-log.js +268 -0
  206. package/src/events-cli.js +102 -0
  207. package/src/events.js +113 -0
  208. package/src/export.js +119 -0
  209. package/src/feedback.js +246 -0
  210. package/src/github.js +551 -0
  211. package/src/hosted/LICENSE.md +105 -0
  212. package/src/hosted/access-cli.js +78 -0
  213. package/src/hosted/admin.js +682 -0
  214. package/src/hosted/agent-approvals.js +163 -0
  215. package/src/hosted/agent-policy.js +189 -0
  216. package/src/hosted/app.js +510 -0
  217. package/src/hosted/auth-routes.js +540 -0
  218. package/src/hosted/auth.js +536 -0
  219. package/src/hosted/backup.js +483 -0
  220. package/src/hosted/board-context.js +35 -0
  221. package/src/hosted/board-routes.js +486 -0
  222. package/src/hosted/board-view.js +94 -0
  223. package/src/hosted/clean-worker.js +24 -0
  224. package/src/hosted/cleaner.js +89 -0
  225. package/src/hosted/comments.js +355 -0
  226. package/src/hosted/console-routes.js +401 -0
  227. package/src/hosted/cookies.js +18 -0
  228. package/src/hosted/crypto.js +80 -0
  229. package/src/hosted/db.js +496 -0
  230. package/src/hosted/device-routes.js +175 -0
  231. package/src/hosted/device.js +233 -0
  232. package/src/hosted/events.js +63 -0
  233. package/src/hosted/guards.js +172 -0
  234. package/src/hosted/guest-view.js +65 -0
  235. package/src/hosted/http.js +39 -0
  236. package/src/hosted/import.js +300 -0
  237. package/src/hosted/mail.js +73 -0
  238. package/src/hosted/mcp-http.js +71 -0
  239. package/src/hosted/membership.js +114 -0
  240. package/src/hosted/operator.js +478 -0
  241. package/src/hosted/ops-cli.js +345 -0
  242. package/src/hosted/pages.js +146 -0
  243. package/src/hosted/permissions.js +113 -0
  244. package/src/hosted/presence.js +372 -0
  245. package/src/hosted/projects.js +124 -0
  246. package/src/hosted/public.js +109 -0
  247. package/src/hosted/push.js +622 -0
  248. package/src/hosted/quotas.js +115 -0
  249. package/src/hosted/ratelimit.js +96 -0
  250. package/src/hosted/rebind.js +316 -0
  251. package/src/hosted/retention.js +128 -0
  252. package/src/hosted/server.js +260 -0
  253. package/src/hosted/setup-link.js +64 -0
  254. package/src/hosted/setup.js +323 -0
  255. package/src/hosted/shares.js +444 -0
  256. package/src/hosted/tailscale.js +204 -0
  257. package/src/hosted/tenancy.js +301 -0
  258. package/src/hosted/token-api.js +455 -0
  259. package/src/hosted/token-events.js +60 -0
  260. package/src/hosted/tokens.js +57 -0
  261. package/src/hosted/turnstile.js +49 -0
  262. package/src/hosted/usercontent.js +154 -0
  263. package/src/hosted/zip.js +155 -0
  264. package/src/invocation.js +58 -0
  265. package/src/journey-paths.js +282 -0
  266. package/src/journeys.js +189 -0
  267. package/src/links.js +106 -0
  268. package/src/local-projects.js +182 -0
  269. package/src/lock.js +193 -0
  270. package/src/mcp.js +417 -0
  271. package/src/mentions.js +151 -0
  272. package/src/net.js +15 -0
  273. package/src/notify/config.js +275 -0
  274. package/src/notify/index.js +301 -0
  275. package/src/notify/lease.js +73 -0
  276. package/src/notify/messages.js +212 -0
  277. package/src/notify/run.js +174 -0
  278. package/src/pixeldiff.js +49 -0
  279. package/src/png.js +143 -0
  280. package/src/reactions.js +111 -0
  281. package/src/remote/cli.js +133 -0
  282. package/src/remote/credentials.js +62 -0
  283. package/src/remote/ids.js +12 -0
  284. package/src/remote/login.js +155 -0
  285. package/src/remote/remote-mcp.js +237 -0
  286. package/src/remote/remote-watch.js +291 -0
  287. package/src/remote/remote.js +313 -0
  288. package/src/remote/share-state.js +78 -0
  289. package/src/remote/share.js +369 -0
  290. package/src/remote/terminal.js +13 -0
  291. package/src/serve-state.js +64 -0
  292. package/src/server/app.js +824 -0
  293. package/src/server/devices.js +186 -0
  294. package/src/server/local-console.js +28 -0
  295. package/src/server/local-user.js +59 -0
  296. package/src/server/pairing-routes.js +159 -0
  297. package/src/server/share-routes.js +89 -0
  298. package/src/server/tailscale.js +97 -0
  299. package/src/sitemap.js +431 -0
  300. package/src/store.js +983 -0
  301. package/src/tokens.js +11 -0
  302. package/src/update/routes.js +245 -0
  303. package/src/watch.js +403 -0
  304. package/src/which.js +58 -0
@@ -0,0 +1,483 @@
1
+ // `shapewall hosted backup` and `restore`: a consistent copy of a hosted
2
+ // server's data folder, and putting one back.
3
+ //
4
+ // A backup is one folder, <to>/shapewall-backup-<UTC time>/ (0700):
5
+ // shapewall.db the database through VACUUM INTO: one consistent snapshot,
6
+ // taken while the server runs (WAL readers never block)
7
+ // files/ pushed captures, copied after the snapshot
8
+ // secret.key the signing key (0600), when the server made one
9
+ // quotas.json the operator's quotas, when there is one
10
+ // backup.json what it holds: schema version, row counts, the database's
11
+ // SHA-256, file count and bytes
12
+ // A SQLite write lock spans only the database snapshot and the listing of
13
+ // its board folders; the files are copied after it is released, so pushes and
14
+ // metadata writes keep working. A board folder that a later push swapped is
15
+ // copied again until it holds still, and files or folders that vanish while
16
+ // copying (retention, deletions) are skipped; backup.json lists both.
17
+ // Open pushes (staging/) are not backed up; they would expire anyway.
18
+ //
19
+ // A backup holds everything: accounts, passkey public keys, comments and
20
+ // captures, and the signing key. Keep it as private as the server itself.
21
+ //
22
+ // Restore copies a backup into a new, empty data folder, checks the database
23
+ // against backup.json (SHA-256 and PRAGMA integrity_check) and runs any newer
24
+ // migrations. `--check` is the restore drill: restore into a scratch folder
25
+ // next to the backup, start a server on it, check its health and its rows,
26
+ // then remove the scratch folder.
27
+
28
+ import fs from 'node:fs'
29
+ import path from 'node:path'
30
+ import http from 'node:http'
31
+ import { createHash, randomUUID } from 'node:crypto'
32
+ import { openDatabase, quietSqliteWarning } from './db.js'
33
+ import { ID_PATTERNS } from './crypto.js'
34
+ import { ID_RE } from '../config.js'
35
+
36
+ export const BACKUP_FORMAT = 'shapewall-hosted-backup'
37
+ const BACKUP_DIR_RE = /^shapewall-backup-\d{8}T\d{6}Z(?:-\d+)?$/
38
+ const COUNTED = ['users', 'passkeys', 'workspaces', 'memberships', 'projects', 'boards', 'comments', 'api_tokens', 'shares']
39
+
40
+ export class BackupError extends Error {}
41
+
42
+ const stampOf = (date) =>
43
+ date
44
+ .toISOString()
45
+ .replace(/[-:]/g, '')
46
+ .replace(/\.\d+Z$/, 'Z')
47
+
48
+ function sha256File(file) {
49
+ const hash = createHash('sha256')
50
+ const fd = fs.openSync(file, 'r')
51
+ const buffer = Buffer.alloc(1024 * 1024)
52
+ try {
53
+ let read
54
+ while ((read = fs.readSync(fd, buffer, 0, buffer.length, null)) > 0) hash.update(buffer.subarray(0, read))
55
+ } finally {
56
+ fs.closeSync(fd)
57
+ }
58
+ return hash.digest('hex')
59
+ }
60
+
61
+ async function openRaw(file, options = {}) {
62
+ quietSqliteWarning()
63
+ const { DatabaseSync } = await import('node:sqlite')
64
+ const conn = new DatabaseSync(file, options)
65
+ conn.exec('PRAGMA busy_timeout = 5000')
66
+ return conn
67
+ }
68
+
69
+ // Whole-database work on a copy (snapshot, counts, the drill): not a request
70
+ // path, so not through tenant() (src/hosted/tenancy.js).
71
+ function counts(conn) {
72
+ const out = {}
73
+ for (const table of COUNTED) out[table] = conn.prepare(`SELECT COUNT(*) AS n FROM ${table}`).get().n
74
+ return out
75
+ }
76
+
77
+ // Copies a folder tree; a folder that disappears while it is copied (a push
78
+ // commit swaps board folders) is tried again from the start.
79
+ function copyTree(from, to, stats, attempt = 0) {
80
+ const before = { ...stats }
81
+ let entries
82
+ try {
83
+ entries = fs.readdirSync(from, { withFileTypes: true })
84
+ } catch (error) {
85
+ if (error.code === 'ENOENT') return
86
+ throw error
87
+ }
88
+ fs.mkdirSync(to, { recursive: true, mode: 0o700 })
89
+ try {
90
+ for (const entry of entries) {
91
+ const source = path.join(from, entry.name)
92
+ const target = path.join(to, entry.name)
93
+ if (entry.isDirectory()) copyTree(source, target, stats)
94
+ else if (entry.isSymbolicLink()) throw new BackupError('backup files must not contain symbolic links')
95
+ else if (entry.isFile()) {
96
+ fs.copyFileSync(source, target, fs.constants.COPYFILE_FICLONE)
97
+ fs.chmodSync(target, 0o600)
98
+ stats.files += 1
99
+ stats.bytes += fs.statSync(target).size
100
+ }
101
+ }
102
+ } catch (error) {
103
+ if (error.code !== 'ENOENT' || attempt >= 3) throw error
104
+ fs.rmSync(to, { recursive: true, force: true })
105
+ Object.assign(stats, before)
106
+ copyTree(from, to, stats, attempt + 1)
107
+ }
108
+ }
109
+
110
+ // The identity of a board folder: a push commit swaps in a new folder by
111
+ // rename, so a changed identity means the folder was replaced.
112
+ function folderId(dir) {
113
+ try {
114
+ const st = fs.statSync(dir, { bigint: true })
115
+ return `${st.dev}:${st.ino}:${st.birthtimeNs}`
116
+ } catch (error) {
117
+ if (error.code === 'ENOENT') return null
118
+ throw error
119
+ }
120
+ }
121
+
122
+ const MISSING_RETRIES = 5
123
+ const MISSING_PAUSE_MS = 50
124
+ const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
125
+
126
+ // Copies one board folder after the lock is released. Files that vanish while
127
+ // copying are skipped and listed; a folder swapped mid-copy is copied again.
128
+ // `locked` is the folder's identity read under the snapshot's lock: a push
129
+ // that committed after the snapshot but before this copy started shows up as
130
+ // a different identity. A push commit renames the old folder away before
131
+ // renaming the new one in, so a missing folder is looked for again a few
132
+ // times; it only counts as deleted once the live database has no row for it.
133
+ async function copyBoard(from, to, stats, rel, { locked, rowExists }) {
134
+ let changed = false
135
+ for (let attempt = 0, misses = 0; ;) {
136
+ const id = folderId(from)
137
+ if (!id) {
138
+ if (misses < MISSING_RETRIES) {
139
+ misses += 1
140
+ await pause(MISSING_PAUSE_MS)
141
+ continue
142
+ }
143
+ if (!rowExists()) return { vanished: true }
144
+ return { changed, missing: [`${rel}/`] }
145
+ }
146
+ if (id !== locked) changed = true
147
+ const local = { files: 0, bytes: 0, missing: [] }
148
+ fs.rmSync(to, { recursive: true, force: true })
149
+ copyLive(from, to, local, rel)
150
+ if (folderId(from) === id || attempt >= 3) {
151
+ stats.files += local.files
152
+ stats.bytes += local.bytes
153
+ return { changed: changed || attempt > 0, missing: local.missing }
154
+ }
155
+ attempt += 1
156
+ }
157
+ }
158
+
159
+ function copyLive(from, to, stats, rel) {
160
+ let entries
161
+ try {
162
+ entries = fs.readdirSync(from, { withFileTypes: true })
163
+ } catch (error) {
164
+ if (error.code !== 'ENOENT') throw error
165
+ stats.missing.push(`${rel}/`)
166
+ return
167
+ }
168
+ fs.mkdirSync(to, { recursive: true, mode: 0o700 })
169
+ for (const entry of entries) {
170
+ const source = path.join(from, entry.name)
171
+ const target = path.join(to, entry.name)
172
+ if (entry.isDirectory()) copyLive(source, target, stats, `${rel}/${entry.name}`)
173
+ else if (entry.isSymbolicLink()) throw new BackupError('backup files must not contain symbolic links')
174
+ else if (entry.isFile()) {
175
+ try {
176
+ fs.copyFileSync(source, target, fs.constants.COPYFILE_FICLONE)
177
+ } catch (error) {
178
+ if (error.code !== 'ENOENT') throw error
179
+ stats.missing.push(`${rel}/${entry.name}`)
180
+ continue
181
+ }
182
+ fs.chmodSync(target, 0o600)
183
+ stats.files += 1
184
+ stats.bytes += fs.statSync(target).size
185
+ }
186
+ }
187
+ }
188
+
189
+ function fileDigests(dir) {
190
+ const files = {}
191
+ const walk = (folder) => {
192
+ for (const entry of fs.readdirSync(folder, { withFileTypes: true })) {
193
+ const file = path.join(folder, entry.name)
194
+ if (entry.isSymbolicLink()) throw new BackupError('backup files must not contain symbolic links')
195
+ if (entry.isDirectory()) walk(file)
196
+ else if (entry.isFile() && file !== path.join(dir, 'backup.json'))
197
+ files[path.relative(dir, file).split(path.sep).join('/')] = sha256File(file)
198
+ }
199
+ }
200
+ walk(dir)
201
+ return files
202
+ }
203
+
204
+ function copySmall(from, to) {
205
+ if (!fs.existsSync(from)) return false
206
+ fs.copyFileSync(from, to)
207
+ fs.chmodSync(to, 0o600)
208
+ return true
209
+ }
210
+
211
+ export async function backup({ dataDir, to, now = new Date(), keepDays = null }) {
212
+ const source = path.resolve(dataDir)
213
+ const dbFile = path.join(source, 'shapewall.db')
214
+ if (!fs.existsSync(dbFile)) throw new BackupError(`no hosted database at ${dbFile} (is --data right?)`)
215
+ const root = path.resolve(to)
216
+ if (root === source || root.startsWith(source + path.sep)) throw new BackupError('back up to a folder outside the data folder')
217
+ fs.mkdirSync(root, { recursive: true, mode: 0o700 })
218
+ let name = `shapewall-backup-${stampOf(now)}`
219
+ for (let n = 2; fs.existsSync(path.join(root, name)); n += 1) name = `shapewall-backup-${stampOf(now)}-${n}`
220
+ const dir = path.join(root, name)
221
+ const partial = `${dir}.${randomUUID()}.partial`
222
+ fs.mkdirSync(partial, { mode: 0o700 })
223
+ try {
224
+ const conn = await openRaw(dbFile)
225
+ let schema
226
+ let rows
227
+ let boards
228
+ const stats = { files: 0, bytes: 0 }
229
+ try {
230
+ // Push commits, retention and deletions all take the same SQLite write
231
+ // lock. Hold it only for the snapshot and the board list; the files are
232
+ // copied after it is released so the server's writes do not wait.
233
+ conn.exec('BEGIN IMMEDIATE')
234
+ const reader = await openRaw(dbFile, { readOnly: true })
235
+ try {
236
+ reader.prepare('VACUUM INTO ?').run(path.join(partial, 'shapewall.db'))
237
+ } finally {
238
+ reader.close()
239
+ }
240
+ fs.chmodSync(path.join(partial, 'shapewall.db'), 0o600)
241
+ const copy = await openRaw(path.join(partial, 'shapewall.db'), { readOnly: true })
242
+ try {
243
+ schema = copy.prepare('PRAGMA user_version').get().user_version
244
+ rows = counts(copy)
245
+ boards = copy.prepare('SELECT workspace_id, project_id, board_id FROM boards ORDER BY workspace_id, project_id, board_id').all()
246
+ } finally {
247
+ copy.close()
248
+ }
249
+ // Each board folder's identity while the lock is still held, so a push
250
+ // that commits before its folder is copied is noticed.
251
+ for (const row of boards) {
252
+ if (!ID_PATTERNS.workspace.test(row.workspace_id) || !ID_PATTERNS.project.test(row.project_id) || !ID_RE.test(row.board_id))
253
+ continue
254
+ row.folderId = folderId(path.join(source, 'files', row.workspace_id, row.project_id, row.board_id))
255
+ }
256
+ conn.exec('COMMIT')
257
+ } finally {
258
+ conn.close()
259
+ }
260
+ const vanished = []
261
+ const changed = []
262
+ const missing = []
263
+ // The live database, read only: is a board whose folder is gone deleted?
264
+ const live = await openRaw(dbFile, { readOnly: true })
265
+ try {
266
+ const boardRow = live.prepare('SELECT 1 FROM boards WHERE workspace_id = ? AND project_id = ? AND board_id = ?')
267
+ for (const row of boards) {
268
+ const rel = `${row.workspace_id}/${row.project_id}/${row.board_id}`
269
+ if (!ID_PATTERNS.workspace.test(row.workspace_id) || !ID_PATTERNS.project.test(row.project_id) || !ID_RE.test(row.board_id))
270
+ continue
271
+ const result = await copyBoard(path.join(source, 'files', rel), path.join(partial, 'files', rel), stats, rel, {
272
+ locked: row.folderId,
273
+ rowExists: () => Boolean(boardRow.get(row.workspace_id, row.project_id, row.board_id)),
274
+ })
275
+ if (result.vanished) vanished.push(rel)
276
+ else {
277
+ if (result.changed) changed.push(rel)
278
+ missing.push(...result.missing)
279
+ }
280
+ }
281
+ } finally {
282
+ live.close()
283
+ }
284
+ if (vanished.length || changed.length || missing.length) stats.skipped = { vanished, changed, missing }
285
+ const { loadCredentials } = await import('./server.js')
286
+ loadCredentials()
287
+ let secret
288
+ if (process.env.SHAPEWALL_HOSTED_SECRET) {
289
+ fs.writeFileSync(path.join(partial, 'secret.key'), `${process.env.SHAPEWALL_HOSTED_SECRET}\n`, { mode: 0o600 })
290
+ secret = true
291
+ } else secret = copySmall(path.join(source, 'secret.key'), path.join(partial, 'secret.key'))
292
+ const quotas = copySmall(path.join(source, 'quotas.json'), path.join(partial, 'quotas.json'))
293
+ const manifest = {
294
+ format: BACKUP_FORMAT,
295
+ version: 1,
296
+ createdAt: now.toISOString(),
297
+ source,
298
+ schema,
299
+ counts: rows,
300
+ db: {
301
+ file: 'shapewall.db',
302
+ bytes: fs.statSync(path.join(partial, 'shapewall.db')).size,
303
+ sha256: sha256File(path.join(partial, 'shapewall.db')),
304
+ },
305
+ files: stats,
306
+ hashes: fileDigests(partial),
307
+ secret,
308
+ quotas,
309
+ }
310
+ fs.writeFileSync(path.join(partial, 'backup.json'), `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 })
311
+ fs.renameSync(partial, dir)
312
+ const pruned = keepDays ? pruneBackups(root, { keepDays, now, keep: dir }) : []
313
+ return { dir, manifest, pruned }
314
+ } catch (error) {
315
+ fs.rmSync(partial, { recursive: true, force: true })
316
+ throw error
317
+ }
318
+ }
319
+
320
+ function readManifest(dir) {
321
+ try {
322
+ const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'backup.json'), 'utf8'))
323
+ return manifest?.format === BACKUP_FORMAT ? manifest : null
324
+ } catch {
325
+ return null
326
+ }
327
+ }
328
+
329
+ // Backups older than keepDays in `root`. Only folders this tool wrote (the
330
+ // name pattern and a readable backup.json) are ever removed.
331
+ export function pruneBackups(root, { keepDays, now = new Date(), keep = null }) {
332
+ const cutoff = now.getTime() - keepDays * 86_400_000
333
+ const removed = []
334
+ for (const name of fs.readdirSync(root)) {
335
+ if (!BACKUP_DIR_RE.test(name)) continue
336
+ const dir = path.join(root, name)
337
+ if (dir === keep) continue
338
+ const manifest = readManifest(dir)
339
+ const created = Date.parse(manifest?.createdAt || '')
340
+ if (!manifest || !Number.isFinite(created) || created >= cutoff) continue
341
+ fs.rmSync(dir, { recursive: true, force: true })
342
+ removed.push(dir)
343
+ }
344
+ return removed
345
+ }
346
+
347
+ // A backup folder, or a folder of backups (the newest one is used).
348
+ export function findBackup(from) {
349
+ const dir = path.resolve(from)
350
+ if (readManifest(dir)) return dir
351
+ let names = []
352
+ try {
353
+ names = fs.readdirSync(dir).filter((name) => BACKUP_DIR_RE.test(name) && readManifest(path.join(dir, name)))
354
+ } catch {
355
+ throw new BackupError(`no backup at ${dir}`)
356
+ }
357
+ if (!names.length) throw new BackupError(`no backup in ${dir} (looked for shapewall-backup-*/backup.json)`)
358
+ return path.join(dir, names.sort().at(-1))
359
+ }
360
+
361
+ export async function restore({ from, data }) {
362
+ const dir = findBackup(from)
363
+ const manifest = readManifest(dir)
364
+ const target = path.resolve(data)
365
+ if (target === dir || target.startsWith(dir + path.sep)) throw new BackupError('restore outside the backup folder')
366
+ if (fs.existsSync(target) && fs.readdirSync(target).length)
367
+ throw new BackupError(`${target} is not empty: restore into a new folder, then point the server at it`)
368
+ if (manifest.db?.file !== 'shapewall.db' || !manifest.files || manifest.version !== 1)
369
+ throw new BackupError('unsupported backup manifest')
370
+ if (manifest.hashes) {
371
+ const actual = fileDigests(dir)
372
+ if (JSON.stringify(Object.entries(actual).sort()) !== JSON.stringify(Object.entries(manifest.hashes).sort()))
373
+ throw new BackupError('backup files do not match their checksums (damaged copy)')
374
+ }
375
+ const dbCopy = path.join(dir, 'shapewall.db')
376
+ if (sha256File(dbCopy) !== manifest.db.sha256)
377
+ throw new BackupError('the database in this backup does not match its backup.json (damaged copy)')
378
+ fs.mkdirSync(target, { recursive: true, mode: 0o700 })
379
+ fs.chmodSync(target, 0o700)
380
+ try {
381
+ fs.copyFileSync(dbCopy, path.join(target, 'shapewall.db'))
382
+ fs.chmodSync(path.join(target, 'shapewall.db'), 0o600)
383
+ const check = await openRaw(path.join(target, 'shapewall.db'))
384
+ let integrity
385
+ try {
386
+ integrity = check.prepare('PRAGMA integrity_check').get()
387
+ } finally {
388
+ check.close()
389
+ }
390
+ if (Object.values(integrity)[0] !== 'ok') throw new BackupError(`integrity check failed: ${Object.values(integrity)[0]}`)
391
+ const stats = { files: 0, bytes: 0 }
392
+ copyTree(path.join(dir, 'files'), path.join(target, 'files'), stats)
393
+ if (stats.files !== manifest.files.files)
394
+ throw new BackupError(`expected ${manifest.files.files} files, copied ${stats.files} (damaged copy?)`)
395
+ copySmall(path.join(dir, 'secret.key'), path.join(target, 'secret.key'))
396
+ copySmall(path.join(dir, 'quotas.json'), path.join(target, 'quotas.json'))
397
+ // Newer migrations, when this build is newer than the backup.
398
+ const conn = await openDatabase(path.join(target, 'shapewall.db'))
399
+ const schema = conn.prepare('PRAGMA user_version').get().user_version
400
+ const rows = counts(conn)
401
+ conn.close()
402
+ return { from: dir, data: target, manifest, schema, counts: rows, files: stats }
403
+ } catch (error) {
404
+ fs.rmSync(target, { recursive: true, force: true })
405
+ throw error
406
+ }
407
+ }
408
+
409
+ // The restore drill: restore into a scratch folder beside the backup (on the
410
+ // same disk, not a RAM /tmp), start a server on a free loopback port, check
411
+ // /api/health, the row counts and that every board's folder is there, stop
412
+ // it and remove the scratch folder. Returns { ok, checks: [...] }.
413
+ export async function restoreDrill({ from, scratch = null, out = () => {} }) {
414
+ const dir = findBackup(from)
415
+ const scratchDir = path.resolve(scratch || path.join(path.dirname(dir), `.restore-drill-${process.pid}-${Date.now()}`))
416
+ if (fs.existsSync(scratchDir)) throw new BackupError('restore drill scratch folder already exists; choose a new folder')
417
+ const checks = []
418
+ const check = (name, ok, detail = '') => {
419
+ checks.push({ name, ok, detail })
420
+ out(`${ok ? 'ok ' : 'FAIL'} ${name}${detail ? `: ${detail}` : ''}`)
421
+ }
422
+ let server = null
423
+ let app = null
424
+ let conn = null
425
+ try {
426
+ const restored = await restore({ from: dir, data: scratchDir })
427
+ check('restore', true, `${restored.files.files} files, schema ${restored.schema}`)
428
+ for (const table of COUNTED) {
429
+ const expected = restored.manifest.counts?.[table]
430
+ if (expected === undefined) continue
431
+ check(`rows in ${table}`, restored.counts[table] === expected, `${restored.counts[table]} (backup says ${expected})`)
432
+ }
433
+ conn = await openDatabase(path.join(scratchDir, 'shapewall.db'))
434
+ const boards = conn.prepare('SELECT workspace_id, project_id, board_id FROM boards').all()
435
+ // Boards deleted after the snapshot, while files were copied, are listed.
436
+ const vanished = new Set(restored.manifest.files.skipped?.vanished || [])
437
+ const missing = boards.filter(
438
+ (row) =>
439
+ !vanished.has(`${row.workspace_id}/${row.project_id}/${row.board_id}`) &&
440
+ !fs.existsSync(path.join(scratchDir, 'files', row.workspace_id, row.project_id, row.board_id)),
441
+ )
442
+ check(
443
+ 'board folders',
444
+ !missing.length,
445
+ missing.length
446
+ ? `${missing.length} of ${boards.length} missing`
447
+ : `${boards.length - vanished.size} present${vanished.size ? `, ${vanished.size} deleted while backing up` : ''}`,
448
+ )
449
+ const { createHostedApp } = await import('./app.js')
450
+ const { readSecret } = await import('./server.js')
451
+ server = http.createServer((req, res) => app(req, res))
452
+ await new Promise((resolve, reject) => {
453
+ server.once('error', reject)
454
+ server.listen(0, '127.0.0.1', resolve)
455
+ })
456
+ const port = server.address().port
457
+ app = createHostedApp({
458
+ db: conn,
459
+ dataDir: scratchDir,
460
+ origin: `http://localhost:${port}`,
461
+ secret: readSecret(scratchDir),
462
+ sweepEveryMs: 0,
463
+ })
464
+ const health = await new Promise((resolve) => {
465
+ http
466
+ .get({ host: '127.0.0.1', port, path: '/api/health' }, (res) => {
467
+ let body = ''
468
+ res.on('data', (chunk) => (body += chunk))
469
+ res.on('end', () => resolve({ status: res.statusCode, body }))
470
+ })
471
+ .on('error', (error) => resolve({ status: 0, body: error.message }))
472
+ })
473
+ check('server starts and answers /api/health', health.status === 200 && /"ok":true/.test(health.body), `HTTP ${health.status}`)
474
+ } catch (error) {
475
+ check('restore drill', false, error.message)
476
+ } finally {
477
+ if (server) await new Promise((resolve) => server.close(resolve))
478
+ if (app) await app.locals.close()
479
+ conn?.close()
480
+ fs.rmSync(scratchDir, { recursive: true, force: true })
481
+ }
482
+ return { ok: checks.every((item) => item.ok), backup: dir, checks }
483
+ }
@@ -0,0 +1,35 @@
1
+ // Boards as the board API and the token API load them (src/hosted/app.js):
2
+ // a board's row, config and comments; the view a hosted board shows; and
3
+ // its images.
4
+
5
+ import { boardView } from '../server/app.js'
6
+ import { publicBoardView } from './board-view.js'
7
+ import { createCommentStore } from './comments.js'
8
+ import { HttpError } from './http.js'
9
+ import { boardConfig, createProjectStore } from './projects.js'
10
+
11
+ export function createBoardContext({ db, dataDir }) {
12
+ function boardContext(t, project, boardId) {
13
+ const row = createProjectStore(t, { db, dataDir }).board(project.id, boardId)
14
+ if (!row) throw new HttpError(404, 'unknown_board', 'No such board.')
15
+ const config = boardConfig(dataDir, t.workspaceId, project.id, row)
16
+ return { row, config, board: config.boards[0], comments: createCommentStore(t, project.id, row.board_id) }
17
+ }
18
+
19
+ return boardContext
20
+ }
21
+
22
+ export function hostedBoardView(ctx) {
23
+ const view = publicBoardView(boardView(ctx.config, ctx.board), ctx.board)
24
+ view.commentCounts = ctx.comments.counts()
25
+ view.capture = { running: false, state: 'idle', startedAt: null, finishedAt: null, done: null, total: null, failed: null, error: null }
26
+ view.pushedAt = ctx.row.pushed_at
27
+ return view
28
+ }
29
+
30
+ export function sendPng(res, file, cacheable) {
31
+ res.setHeader('Cache-Control', cacheable ? 'private, max-age=604800, immutable' : 'private, no-cache')
32
+ res.setHeader('Cross-Origin-Resource-Policy', 'same-origin')
33
+ res.setHeader('Content-Security-Policy', "default-src 'none'; sandbox")
34
+ res.type('png').sendFile(file, { dotfiles: 'allow' })
35
+ }