vowel 0.6.3 → 0.7.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/README.md +23 -17
  2. package/bin.js +87 -30
  3. package/cli.js +15 -0
  4. package/config.js +102 -16
  5. package/extractDate.js +70 -6
  6. package/index.js +14 -5
  7. package/package.json +42 -7
  8. package/plugins/assets/index.js +65 -0
  9. package/plugins/atproto/index.js +132 -0
  10. package/plugins/atproto/password.js +71 -0
  11. package/plugins/atproto/publish.js +406 -0
  12. package/plugins/atproto/record.js +144 -0
  13. package/plugins/atproto/rkey.js +54 -0
  14. package/plugins/deploy-cloudflare/index.js +92 -0
  15. package/plugins/fonts/index.js +18 -21
  16. package/plugins/html/assets/bundle-BeK1Hptv.css +21 -0
  17. package/plugins/html/bundle/assets/index-BeK1Hptv.css +21 -0
  18. package/plugins/html/bundle/index.js +21145 -0
  19. package/plugins/html/editor/EditButton.svelte +152 -0
  20. package/plugins/html/editor/Editor.svelte +51 -0
  21. package/plugins/html/editor/Overlays.svelte +6 -0
  22. package/plugins/html/editor/SettingsPanel.svelte +397 -0
  23. package/plugins/html/editor/Toolbar.svelte +644 -0
  24. package/plugins/html/editor/create-session.js +250 -0
  25. package/plugins/html/editor/directives.js +128 -0
  26. package/plugins/html/editor/frontmatter.js +97 -0
  27. package/plugins/html/editor/ingest.js +339 -0
  28. package/plugins/html/editor/main.js +94 -0
  29. package/plugins/html/editor/nodes/Alert.svelte +21 -0
  30. package/plugins/html/editor/nodes/Blockquote.svelte +9 -0
  31. package/plugins/html/editor/nodes/CodeBlock.svelte +19 -0
  32. package/plugins/html/editor/nodes/Embed.svelte +19 -0
  33. package/plugins/html/editor/nodes/Emphasis.svelte +11 -0
  34. package/plugins/html/editor/nodes/Heading.svelte +15 -0
  35. package/plugins/html/editor/nodes/Highlight.svelte +11 -0
  36. package/plugins/html/editor/nodes/Image.svelte +18 -0
  37. package/plugins/html/editor/nodes/InlineCode.svelte +11 -0
  38. package/plugins/html/editor/nodes/Link.svelte +17 -0
  39. package/plugins/html/editor/nodes/List.svelte +15 -0
  40. package/plugins/html/editor/nodes/ListItem.svelte +9 -0
  41. package/plugins/html/editor/nodes/Page.svelte +21 -0
  42. package/plugins/html/editor/nodes/Paragraph.svelte +9 -0
  43. package/plugins/html/editor/nodes/Strikethrough.svelte +11 -0
  44. package/plugins/html/editor/nodes/Strong.svelte +11 -0
  45. package/plugins/html/editor/nodes/ThematicBreak.svelte +9 -0
  46. package/plugins/html/editor/nodes/index.js +18 -0
  47. package/plugins/html/editor/rollup.config.js +35 -0
  48. package/plugins/html/editor/serialize.js +157 -0
  49. package/plugins/html/editor/settings-file.js +83 -0
  50. package/plugins/html/editor/socket.js +202 -0
  51. package/plugins/html/editor/source.js +43 -0
  52. package/plugins/html/image.js +104 -0
  53. package/plugins/html/index.js +760 -0
  54. package/plugins/html/partials.js +299 -0
  55. package/plugins/html/socialLinks.js +67 -0
  56. package/plugins/html/writeRules.js +377 -0
  57. package/plugins/icons/index.js +2 -6
  58. package/plugins/images/index.js +78 -55
  59. package/plugins/markdown/brandColors.js +10 -0
  60. package/plugins/markdown/colorScheme.js +64 -0
  61. package/plugins/markdown/emoji.js +30 -0
  62. package/plugins/markdown/frontmatter.js +253 -0
  63. package/plugins/markdown/index.js +287 -1274
  64. package/plugins/markdown/links.js +97 -0
  65. package/plugins/markdown/metadata.js +308 -0
  66. package/plugins/markdown/readRules.js +143 -0
  67. package/plugins/markdown/typography.js +109 -0
  68. package/plugins/markdown/visitor.js +143 -0
  69. package/plugins/publish/index.js +34 -0
  70. package/plugins/robots/agents.js +79 -0
  71. package/plugins/robots/index.js +30 -10
  72. package/plugins/robots/robots.js +48 -0
  73. package/plugins/styles/index.js +111 -26
  74. package/plugins/styles/theme.js +116 -0
  75. package/plugins/urls/index.js +78 -0
  76. package/plugins/urls/urlMetadata.js +246 -0
  77. package/plugins/vectors/index.js +5 -9
  78. package/plugins/xml/entryContent.js +103 -0
  79. package/plugins/xml/index.js +123 -50
  80. package/secretPaths.js +170 -0
  81. package/stylesheets/DefaultStyles.css +367 -437
  82. package/stylesheets/ResetStyles.css +2 -38
  83. package/stylesheets/SyntaxHighlightingStyles.css +269 -0
  84. package/stylesheets/TypographyStyles.css +386 -128
  85. package/systemPaths.js +48 -0
  86. package/utils.js +56 -5
  87. package/.github/ISSUE_TEMPLATE/bug_report.md +0 -73
  88. package/.github/ISSUE_TEMPLATE/config.yml +0 -7
  89. package/docs-source/.cache.json +0 -9
  90. package/docs-source/.obsidian/app.json +0 -3
  91. package/docs-source/.obsidian/appearance.json +0 -3
  92. package/docs-source/.obsidian/core-plugins-migration.json +0 -30
  93. package/docs-source/.obsidian/core-plugins.json +0 -20
  94. package/docs-source/.obsidian/workspace.json +0 -168
  95. package/docs-source/.stackblitzrc +0 -3
  96. package/docs-source/.votive.db +0 -0
  97. package/docs-source/about.md +0 -3
  98. package/docs-source/blog/home.md +0 -5
  99. package/docs-source/blog/url-ui.md +0 -21
  100. package/docs-source/blog.md +0 -5
  101. package/docs-source/docs/.votive.db +0 -0
  102. package/docs-source/docs/deploy.md +0 -67
  103. package/docs-source/docs/file-structure.md +0 -31
  104. package/docs-source/docs/folder-settings.md +0 -23
  105. package/docs-source/docs/home.md +0 -55
  106. package/docs-source/docs/images.md +0 -10
  107. package/docs-source/docs/items.md +0 -13
  108. package/docs-source/docs/pages.md +0 -141
  109. package/docs-source/docs/settings.md +0 -4
  110. package/docs-source/docs/styling.md +0 -34
  111. package/docs-source/docs/taxonomies.md +0 -37
  112. package/docs-source/features/cards.md +0 -7
  113. package/docs-source/features/editing.md +0 -7
  114. package/docs-source/features/emoji.md +0 -7
  115. package/docs-source/features/frontmatter.md +0 -7
  116. package/docs-source/features/lists.md +0 -7
  117. package/docs-source/features/navigation.md +0 -7
  118. package/docs-source/features/rich-previews.md +0 -7
  119. package/docs-source/features/robots.md +0 -7
  120. package/docs-source/features/rss.md +0 -7
  121. package/docs-source/features/sitemap.md +0 -7
  122. package/docs-source/features/speed.md +0 -7
  123. package/docs-source/features/static.md +0 -7
  124. package/docs-source/features/taxonomies.md +0 -7
  125. package/docs-source/home.md +0 -42
  126. package/docs-source/output/about.html +0 -1
  127. package/docs-source/output/blog/url-ui.html +0 -8
  128. package/docs-source/output/blog.html +0 -1
  129. package/docs-source/output/default.css +0 -1
  130. package/docs-source/output/docs/deploy.html +0 -34
  131. package/docs-source/output/docs/file-structure.html +0 -24
  132. package/docs-source/output/docs/folder-settings.html +0 -20
  133. package/docs-source/output/docs/images.html +0 -2
  134. package/docs-source/output/docs/items.html +0 -6
  135. package/docs-source/output/docs/pages.html +0 -101
  136. package/docs-source/output/docs/styling.html +0 -18
  137. package/docs-source/output/docs/taxonomies.html +0 -20
  138. package/docs-source/output/docs.html +0 -28
  139. package/docs-source/output/features/cards.html +0 -1
  140. package/docs-source/output/features/editing.html +0 -1
  141. package/docs-source/output/features/emoji.html +0 -1
  142. package/docs-source/output/features/frontmatter.html +0 -1
  143. package/docs-source/output/features/lists.html +0 -1
  144. package/docs-source/output/features/navigation.html +0 -1
  145. package/docs-source/output/features/rich-previews.html +0 -1
  146. package/docs-source/output/features/robots.html +0 -1
  147. package/docs-source/output/features/rss.html +0 -1
  148. package/docs-source/output/features/sitemap.html +0 -1
  149. package/docs-source/output/features/speed.html +0 -1
  150. package/docs-source/output/features/static.html +0 -1
  151. package/docs-source/output/features/taxonomies.html +0 -1
  152. package/docs-source/output/features.html +0 -1
  153. package/docs-source/output/feed.xml +0 -1
  154. package/docs-source/output/index.html +0 -21
  155. package/docs-source/output/reset.css +0 -1
  156. package/docs-source/output/roadmap.html +0 -87
  157. package/docs-source/output/robots.txt +0 -14
  158. package/docs-source/output/sitemap.xml +0 -16
  159. package/docs-source/output/typography.css +0 -1
  160. package/docs-source/roadmap.md +0 -98
  161. package/docs-source/settings.md +0 -13
  162. package/getMetadata.js +0 -41
  163. package/jsconfig.json +0 -10
  164. package/regex.js +0 -36
  165. package/stylesheets/brand-colors.css +0 -119
@@ -0,0 +1,406 @@
1
+ import { createHash } from "node:crypto"
2
+ import { mkdtemp, rm, readFile } from "node:fs/promises"
3
+ import { tmpdir } from "node:os"
4
+ import path from "node:path"
5
+ import sharp from "sharp"
6
+ import votive from "votive"
7
+ import { systemDirectoryFor } from "../../systemPaths.js"
8
+ import { DOCUMENT, PUBLICATION, siteOrigin, publicationURI } from "./record.js"
9
+
10
+ /**
11
+ * `vowel --command atproto-publish`: make the account's records match the
12
+ * site.
13
+ *
14
+ * The PDS is the ledger. The command builds the site, collects the record
15
+ * each page's transform stored (`metadata.atprotoRecord`), lists what the
16
+ * PDS holds, and sends the difference through `applyWrites`: a record the
17
+ * site has and the PDS does not is created, one that differs is updated,
18
+ * and one the PDS has for this site and the site no longer produces is
19
+ * deleted. Nothing is remembered between runs, so clearing a cache
20
+ * cannot cause a resend, a revert publishes, and a coworker sees the
21
+ * same state.
22
+ *
23
+ * **Only this site's records are ever deleted.** `listRecords` returns
24
+ * every record in the collection, including ones another site or app
25
+ * published to the same account. A record is this site's when its `site`
26
+ * is this site's origin. A record at one of our keys with another `site`
27
+ * is a collision; it is reported and left alone.
28
+ *
29
+ * The build uses a throwaway database, so every page is read and every
30
+ * transform runs against current settings. A transform only re-runs when
31
+ * its source is re-read (tasks/1-proposed/transform-reruns-on-stale.md),
32
+ * so a warm database could hold records built from old settings.
33
+ *
34
+ * **The publication goes first.** One `site.standard.publication` per
35
+ * site, at the key record.js derives from the origin: `url` from
36
+ * `domain:`, `name` from the root `name:` (else the homepage's title),
37
+ * and `description` and `icon` from the root's `atproto_publication:`.
38
+ * The icon defaults to `logo:`. It is rendered to a 512px square PNG and
39
+ * its blob reference - CID, type, size - computed here, so an unchanged
40
+ * icon compares equal and is never uploaded again; a changed one is
41
+ * uploaded just before the write that names it. The publication is
42
+ * created and updated, never deleted.
43
+ *
44
+ * The app password comes from `ATPROTO_APP_PASSWORD`, or from a file
45
+ * named `atproto-app-password` in the project's system directory, never
46
+ * from the project. Payload: `{"dryRun": true}` reports what would change
47
+ * and sends nothing, and does not need the password. `{"optional": true}`
48
+ * makes a site with no `atproto_did` a skip rather than an error, for
49
+ * the `publish` command, which runs this after every deploy.
50
+ */
51
+
52
+ const PASSWORD_FILE = "atproto-app-password"
53
+ const PAGE_SIZE = 100
54
+ const BATCH_SIZE = 100
55
+ const TIMEOUT = 30_000
56
+ const ICON_SIZE = 512
57
+ // The lexicon's maxSize for `icon`.
58
+ const ICON_MAX_BYTES = 1_000_000
59
+ const BASE32 = "abcdefghijklmnopqrstuvwxyz234567"
60
+
61
+ /**
62
+ * RFC 4648 base32, lowercase, unpadded: the multibase `b` alphabet.
63
+ * @param {Uint8Array} bytes
64
+ */
65
+ function base32(bytes) {
66
+ const bits = [...bytes].map(byte => byte.toString(2).padStart(8, "0")).join("")
67
+ const padded = bits.padEnd(Math.ceil(bits.length / 5) * 5, "0")
68
+ return padded.match(/.{5}/g).map(chunk => BASE32[parseInt(chunk, 2)]).join("")
69
+ }
70
+
71
+ /**
72
+ * The CID a PDS gives a blob: CIDv1, raw codec (0x55), sha2-256.
73
+ * @param {Uint8Array} bytes
74
+ */
75
+ function blobCID(bytes) {
76
+ const digest = createHash("sha256").update(bytes).digest()
77
+ return "b" + base32(Buffer.concat([Buffer.from([0x01, 0x55, 0x12, 0x20]), digest]))
78
+ }
79
+
80
+ /**
81
+ * The publication's icon: a 512px square PNG, the image contained in it
82
+ * on transparency, and the blob reference a PDS will give it. An SVG is
83
+ * rasterised at a density high enough to stay sharp at that size.
84
+ * @param {Buffer} source
85
+ * @param {string} iconPath - for the error message
86
+ */
87
+ async function iconBlob(source, iconPath) {
88
+ const bytes = await sharp(source, { density: 300 })
89
+ .resize(ICON_SIZE, ICON_SIZE, { fit: "contain", background: { r: 0, g: 0, b: 0, alpha: 0 } })
90
+ .png()
91
+ .toBuffer()
92
+ if (bytes.length > ICON_MAX_BYTES) {
93
+ throw new Error(`atproto: the publication icon ${iconPath} is ${bytes.length} bytes as a ${ICON_SIZE}px PNG; the limit is ${ICON_MAX_BYTES}`)
94
+ }
95
+ const ref = { $type: "blob", ref: { $link: blobCID(bytes) }, mimeType: "image/png", size: bytes.length }
96
+ return { bytes, ref }
97
+ }
98
+
99
+ /**
100
+ * A value with its object keys sorted, so two records compare by content
101
+ * and not by key order.
102
+ * @param {unknown} value
103
+ * @returns {unknown}
104
+ */
105
+ function canonical(value) {
106
+ if (Array.isArray(value)) return value.map(canonical)
107
+ if (!value || typeof value !== "object") return value
108
+ const keys = Object.keys(value).sort()
109
+ return Object.fromEntries(keys.map(key => [key, canonical(value[key])]))
110
+ }
111
+
112
+ /** @param {unknown} a @param {unknown} b */
113
+ function sameRecord(a, b) {
114
+ return JSON.stringify(canonical(a)) === JSON.stringify(canonical(b))
115
+ }
116
+
117
+ /**
118
+ * One XRPC call. An error response throws with the PDS's own error name
119
+ * and message, which is the only useful part of a failed write.
120
+ * @param {string} pds
121
+ * @param {string} nsid
122
+ * `bytes` is a raw body (uploadBlob) of type `mimeType`; `body` is JSON.
123
+ * @param {{ query?: Record<string, string>, body?: unknown, bytes?: Uint8Array, mimeType?: string, token?: string }} [options]
124
+ */
125
+ async function xrpc(pds, nsid, { query, body, bytes, mimeType, token } = {}) {
126
+ const url = new URL(`/xrpc/${nsid}`, pds)
127
+ const params = new URLSearchParams(query ?? {})
128
+ url.search = params.toString()
129
+ const payload = bytes ?? (body === undefined ? undefined : JSON.stringify(body))
130
+ const headers = {
131
+ ...(bytes ? { "content-type": mimeType } : body === undefined ? {} : { "content-type": "application/json" }),
132
+ ...(token ? { authorization: `Bearer ${token}` } : {})
133
+ }
134
+ const response = await fetch(url, {
135
+ method: payload === undefined ? "GET" : "POST",
136
+ headers,
137
+ body: payload,
138
+ signal: AbortSignal.timeout(TIMEOUT)
139
+ })
140
+ const text = await response.text()
141
+ const json = text ? JSON.parse(text) : {}
142
+ if (response.ok) return json
143
+ throw new Error(`${nsid}: ${response.status} ${json.error ?? ""} ${json.message ?? ""}`.trim())
144
+ }
145
+
146
+ /**
147
+ * The PDS that hosts `did`, from its DID document. `did:plc` is resolved
148
+ * through the PLC directory, `did:web` through its own host.
149
+ * @param {string} did
150
+ */
151
+ async function resolvePDS(did) {
152
+ const documentURL = did.startsWith("did:plc:")
153
+ ? `https://plc.directory/${did}`
154
+ : did.startsWith("did:web:")
155
+ ? `https://${decodeURIComponent(did.slice("did:web:".length))}/.well-known/did.json`
156
+ : undefined
157
+ if (!documentURL) throw new Error(`atproto: can't resolve ${did}; set atproto_pds in settings.md`)
158
+
159
+ const response = await fetch(documentURL, { signal: AbortSignal.timeout(TIMEOUT) })
160
+ if (!response.ok) throw new Error(`atproto: resolving ${did} failed: ${response.status}`)
161
+ const document = await response.json()
162
+ const service = (document.service ?? []).find(entry => entry.id === "#atproto_pds" || entry.id === `${did}#atproto_pds`)
163
+ if (!service) throw new Error(`atproto: ${did} names no PDS`)
164
+ return service.serviceEndpoint
165
+ }
166
+
167
+ /**
168
+ * Where the app password is kept: the project's system directory, never
169
+ * the project, so it is neither committed nor deployed.
170
+ * @param {string} sourceFolder
171
+ */
172
+ function passwordFile(sourceFolder) {
173
+ return path.join(systemDirectoryFor(sourceFolder), PASSWORD_FILE)
174
+ }
175
+
176
+ /** @param {string} sourceFolder */
177
+ async function appPassword(sourceFolder) {
178
+ if (process.env.ATPROTO_APP_PASSWORD) return process.env.ATPROTO_APP_PASSWORD
179
+ const file = passwordFile(sourceFolder)
180
+ const text = await readFile(file, "utf-8").catch(() => "")
181
+ if (text.trim()) return text.trim()
182
+ throw new Error(`atproto: no app password. Set ATPROTO_APP_PASSWORD, or put one in ${file}`)
183
+ }
184
+
185
+ /**
186
+ * Every record in `collection`, a page at a time.
187
+ * @param {string} pds
188
+ * @param {string} did
189
+ * @param {string} collection
190
+ * @param {string} [cursor]
191
+ * @returns {Promise<{ uri: string, value: any }[]>}
192
+ */
193
+ async function listAll(pds, did, collection, cursor) {
194
+ const query = { repo: did, collection, limit: String(PAGE_SIZE), ...(cursor ? { cursor } : {}) }
195
+ const page = await xrpc(pds, "com.atproto.repo.listRecords", { query })
196
+ const records = page.records ?? []
197
+ if (!page.cursor || !records.length) return records
198
+ const rest = await listAll(pds, did, collection, page.cursor)
199
+ return [...records, ...rest]
200
+ }
201
+
202
+ /** The last segment of an AT-URI. @param {string} uri */
203
+ const rkeyOf = (uri) => uri.slice(uri.lastIndexOf("/") + 1)
204
+
205
+ /**
206
+ * What the site should hold: every stored record, by rkey, the site's
207
+ * own settings, and the publication's name, description and icon. The
208
+ * icon is read from the build's output - it is a URL path from the root
209
+ * (frontmatter.js resolves it), which is where the output has it. Built
210
+ * into a scratch folder that is removed afterwards.
211
+ * @param {import("votive").VotiveConfig} config
212
+ */
213
+ async function collectRecords(config) {
214
+ const scratch = await mkdtemp(path.join(tmpdir(), "vowel-atproto-"))
215
+ try {
216
+ const site = await votive({
217
+ ...config,
218
+ verbose: false,
219
+ databasePath: path.join(scratch, ".votive.db"),
220
+ targetFolder: path.join(scratch, "output")
221
+ })
222
+ try {
223
+ await (await site.build({ defer: false })).deferred
224
+ const root = site.database.setting.getByFolder("")
225
+ const did = root.lastNonNull("fm_atproto_did")
226
+ const pds = root.lastNonNull("fm_atproto_pds")
227
+ const domain = root.lastNonNull("fm_domain")
228
+ const targets = site.database.target.getAll()
229
+ const pages = targets.filter(target => "atUri" in target.metadata && "atprotoRecord" in target.metadata)
230
+ const records = pages.map(target => ({ uri: target.metadata.atUri, value: target.metadata.atprotoRecord }))
231
+
232
+ const settings = root.lastNonNull("fm_atproto_publication") ?? {}
233
+ const home = site.database.target.get("index.html")
234
+ const name = root.raw("fm_name")?.[0]?.at(-1) ?? (home?.source ? home.metadata.title : undefined)
235
+ const iconPath = settings.icon ?? root.lastNonNull("logo")
236
+ const icon = iconPath ? await readIcon(path.join(scratch, "output"), iconPath) : undefined
237
+ const publication = { name, description: settings.description, iconPath, icon }
238
+ return { did, pds, domain, records, publication }
239
+ } finally {
240
+ await site.close()
241
+ }
242
+ } finally {
243
+ await rm(scratch, { recursive: true, force: true })
244
+ }
245
+ }
246
+
247
+ /**
248
+ * The icon's bytes from the build output. A URL is not fetched: the icon
249
+ * has to be a file in the project.
250
+ * @param {string} output
251
+ * @param {string} iconPath - a URL path from the root, e.g. "/logo.svg"
252
+ */
253
+ async function readIcon(output, iconPath) {
254
+ if (/^[a-z]+:/i.test(iconPath)) {
255
+ throw new Error(`atproto: the publication icon must be a file in the project, not ${iconPath}`)
256
+ }
257
+ const file = path.join(output, ...iconPath.split("/").filter(Boolean))
258
+ return readFile(file).catch(() => {
259
+ throw new Error(`atproto: the publication icon ${iconPath} is not in the built site`)
260
+ })
261
+ }
262
+
263
+ /**
264
+ * The publication's write: create, update or unchanged, against what the
265
+ * account holds at its key.
266
+ * @param {{ uri: string, value: any }[]} remote - the account's publications
267
+ * @param {string} rkey
268
+ * @param {Record<string, unknown>} value
269
+ */
270
+ function planPublication(remote, rkey, value) {
271
+ const existing = remote.find(record => rkeyOf(record.uri) === rkey)
272
+ if (!existing) return { action: "create", rkey, value }
273
+ if (sameRecord(existing.value, value)) return { action: "unchanged", rkey }
274
+ return { action: "update", rkey, value }
275
+ }
276
+
277
+ /**
278
+ * The writes that make `remote` match `local`, plus the collisions.
279
+ * @param {{ uri: string, value: any }[]} local
280
+ * @param {{ uri: string, value: any }[]} remote
281
+ * @param {Set<string>} sites - the `site` values that are this site's
282
+ */
283
+ function planWrites(local, remote, sites) {
284
+ const remoteByKey = new Map(remote.map(record => [rkeyOf(record.uri), record]))
285
+ const localKeys = new Set(local.map(record => rkeyOf(record.uri)))
286
+
287
+ const compared = local.map(record => {
288
+ const rkey = rkeyOf(record.uri)
289
+ const existing = remoteByKey.get(rkey)
290
+ if (!existing) return { action: "create", rkey, value: record.value }
291
+ if (!sites.has(existing.value?.site)) return { action: "collision", rkey, site: existing.value?.site }
292
+ if (sameRecord(existing.value, record.value)) return { action: "unchanged", rkey }
293
+ return { action: "update", rkey, value: record.value }
294
+ })
295
+
296
+ const deletions = remote
297
+ .filter(record => sites.has(record.value?.site) && !localKeys.has(rkeyOf(record.uri)))
298
+ .map(record => ({ action: "delete", rkey: rkeyOf(record.uri) }))
299
+
300
+ return [...compared, ...deletions]
301
+ }
302
+
303
+ /** @param {{ action: string, rkey: string, value?: unknown, collection?: string }} write */
304
+ function applyWrite({ action, rkey, value, collection = DOCUMENT }) {
305
+ const base = { $type: `com.atproto.repo.applyWrites#${action}`, collection, rkey }
306
+ return action === "delete" ? base : { ...base, value }
307
+ }
308
+
309
+ /**
310
+ * @param {{ dryRun?: boolean, optional?: boolean } | undefined} payload
311
+ * @param {{ config: import("votive").VotiveConfig, notify: (message: object) => void }} context
312
+ */
313
+ async function publishToAtproto(payload, { config, notify }) {
314
+ const dryRun = Boolean(payload?.dryRun)
315
+
316
+ notify({ status: "progress", message: "Building..." })
317
+ const { did, pds: configuredPDS, domain, records, publication } = await collectRecords(config)
318
+
319
+ if ((typeof did !== "string" || !did) && payload?.optional) {
320
+ notify({ status: "done", message: "No atproto_did in settings.md; nothing to publish to AT Protocol" })
321
+ return { skipped: true }
322
+ }
323
+ if (typeof did !== "string" || !did) throw new Error("atproto: set atproto_did in the root settings.md")
324
+ const origin = siteOrigin(domain)
325
+ if (!origin) throw new Error("atproto: set domain in the root settings.md; every record carries the site's url")
326
+
327
+ const accounts = new Set(records.map(record => record.uri.split("/")[2]))
328
+ const others = [...accounts].filter(account => account !== did)
329
+ if (others.length) throw new Error(`atproto: pages name ${others.join(", ")}; one site publishes to one account (${did})`)
330
+
331
+ if (typeof publication.name !== "string" || !publication.name) {
332
+ throw new Error("atproto: set name in the root settings.md; the publication needs one")
333
+ }
334
+
335
+ const icon = publication.icon ? await iconBlob(publication.icon, publication.iconPath) : undefined
336
+ const publicationRecord = {
337
+ $type: PUBLICATION,
338
+ url: origin,
339
+ name: publication.name,
340
+ ...(typeof publication.description === "string" && publication.description ? { description: publication.description } : {}),
341
+ ...(icon ? { icon: icon.ref } : {})
342
+ }
343
+ const publicationURL = publicationURI(did, origin)
344
+
345
+ // A document is this site's when its `site` is the publication, or the
346
+ // origin a loose document carried before there was one.
347
+ const sites = new Set([origin, publicationURL, ...records.map(record => record.value.site)])
348
+ const pds = typeof configuredPDS === "string" && configuredPDS ? configuredPDS : await resolvePDS(did)
349
+
350
+ notify({ status: "progress", message: `Comparing ${records.length} records with ${pds}...` })
351
+ const remote = await listAll(pds, did, DOCUMENT)
352
+ const remotePublications = await listAll(pds, did, PUBLICATION)
353
+ const publicationWrite = planPublication(remotePublications, rkeyOf(publicationURL), publicationRecord)
354
+ const plan = planWrites(records, remote, sites)
355
+
356
+ const count = (action) => plan.filter(write => write.action === action).length
357
+ const summary = {
358
+ created: count("create"),
359
+ updated: count("update"),
360
+ deleted: count("delete"),
361
+ unchanged: count("unchanged"),
362
+ collisions: plan.filter(write => write.action === "collision").map(({ rkey, site }) => ({ rkey, site })),
363
+ publication: publicationWrite.action,
364
+ dryRun
365
+ }
366
+ // A command's config comes straight from the caller and may carry no
367
+ // logger; votive() adds one only to the config its hooks receive.
368
+ const log = config.log ?? ((level, message) => console.error(message))
369
+ for (const { rkey, site } of summary.collisions) {
370
+ log("error", `atproto: ${DOCUMENT}/${rkey} belongs to ${site}, not ${origin}; left alone`)
371
+ }
372
+
373
+ const publicationWrites = publicationWrite.action === "unchanged" ? [] : [{ ...publicationWrite, collection: PUBLICATION }]
374
+ const documentWrites = plan.filter(write => ["create", "update", "delete"].includes(write.action))
375
+ const writes = [...publicationWrites, ...documentWrites].map(applyWrite)
376
+ if (dryRun || !writes.length) {
377
+ notify({ status: "done", message: `${dryRun ? "Would send" : "Nothing to send"}: ${JSON.stringify(summary)}` })
378
+ return summary
379
+ }
380
+
381
+ const password = await appPassword(config.sourceFolder)
382
+ const session = await xrpc(pds, "com.atproto.server.createSession", { body: { identifier: did, password } })
383
+ if (session.did !== did) throw new Error(`atproto: the app password is for ${session.did}, not ${did}`)
384
+
385
+ // The icon is uploaded only when the publication is written - which is
386
+ // when its reference changed, or the record is new.
387
+ if (icon && publicationWrites.length) {
388
+ notify({ status: "progress", message: "Uploading the publication icon..." })
389
+ const uploaded = await xrpc(pds, "com.atproto.repo.uploadBlob", { bytes: icon.bytes, mimeType: icon.ref.mimeType, token: session.accessJwt })
390
+ if (uploaded.blob?.ref?.$link !== icon.ref.ref.$link) {
391
+ throw new Error(`atproto: the PDS gave the icon ${uploaded.blob?.ref?.$link}, not the ${icon.ref.ref.$link} computed for it; nothing was written`)
392
+ }
393
+ }
394
+
395
+ const batches = Array.from({ length: Math.ceil(writes.length / BATCH_SIZE) }, (_, index) =>
396
+ writes.slice(index * BATCH_SIZE, (index + 1) * BATCH_SIZE))
397
+ for (const [index, batch] of batches.entries()) {
398
+ notify({ status: "progress", message: `Sending batch ${index + 1} of ${batches.length}...` })
399
+ await xrpc(pds, "com.atproto.repo.applyWrites", { body: { repo: did, writes: batch }, token: session.accessJwt })
400
+ }
401
+
402
+ notify({ status: "done", message: `Published: ${JSON.stringify(summary)}` })
403
+ return summary
404
+ }
405
+
406
+ export { publishToAtproto, planWrites, planPublication, sameRecord, resolvePDS, blobCID, xrpc, passwordFile }
@@ -0,0 +1,144 @@
1
+ import { toText } from "hast-util-to-text"
2
+ import { MARKER } from "../../secretPaths.js"
3
+ import createRkey from "./rkey.js"
4
+
5
+ /**
6
+ * Which pages become records, and what each record holds. No network:
7
+ * everything here is a function of the page and its settings, so it runs
8
+ * in the build, and the publish command only compares and sends.
9
+ */
10
+
11
+ const DOCUMENT = "site.standard.document"
12
+ const PUBLICATION = "site.standard.publication"
13
+
14
+ /** The lexicons vowel can build a record for. */
15
+ const SUPPORTED = [DOCUMENT]
16
+
17
+ /**
18
+ * The page's lexicon and, for a secret page, whether it may use it.
19
+ *
20
+ * `atproto_lexicon:` cascades from settings.md, and a page's own
21
+ * frontmatter can set it too. A page under a secret segment (`##`) is published only when
22
+ * the lexicon was set *inside* its secret scope: in its own frontmatter,
23
+ * or in a settings.md at or below the secret folder. A lexicon inherited
24
+ * from above the secret would publish a hidden page to the network by
25
+ * accident.
26
+ *
27
+ * The settings chain is the page-as-folder chain of the routed path
28
+ * (`blog/post.html` reads `""`, `blog`, `blog/post`), so level `k` holds
29
+ * the first `k` segments. Routing replaces a segment with its hash but
30
+ * keeps the depth, so the secret segment at source index `s` is level
31
+ * `s + 1`, and every level from there down is inside it.
32
+ * @param {{ source?: string, metadata: Record<string, any> }} target
33
+ * @param {{ raw: (label: string) => (unknown[] | null)[] | undefined }} settings
34
+ * @returns {string | undefined}
35
+ */
36
+ function lexiconFor(target, settings) {
37
+ const own = target.metadata.fm_atproto_lexicon
38
+ if (typeof own === "string" && own) return own
39
+
40
+ const levels = settings.raw("fm_atproto_lexicon") ?? []
41
+ const deepest = levels.findLastIndex(values => Array.isArray(values) && values.length > 0)
42
+ if (deepest === -1) return undefined
43
+ const lexicon = levels[deepest].at(-1)
44
+ if (typeof lexicon !== "string" || !lexicon) return undefined
45
+
46
+ const segments = (target.source ?? "").split(/[\\/]/)
47
+ const secret = segments.findIndex(segment => segment.includes(MARKER))
48
+ if (secret !== -1 && deepest < secret + 1) return undefined
49
+ return lexicon
50
+ }
51
+
52
+ /**
53
+ * The site's origin from `domain:`, the same base the html plugin builds
54
+ * the canonical link from. No trailing slash, per the lexicon.
55
+ * @param {unknown} domain
56
+ */
57
+ function siteOrigin(domain) {
58
+ if (typeof domain !== "string" || !domain) return undefined
59
+ const base = domain.startsWith("http") ? domain : `https://${domain}`
60
+ return new URL(base).origin
61
+ }
62
+
63
+ /**
64
+ * The site's publication: one per account and origin, its key derived
65
+ * from the origin the way a document's is from its URL, so its AT-URI is
66
+ * known before it exists. A document's `site` names it, and
67
+ * `/.well-known/site.standard.publication` serves it, which is how the
68
+ * domain claims it. The record itself is built by the publish command
69
+ * (publish.js), because its icon is an uploaded image.
70
+ * @param {string} did
71
+ * @param {string} origin
72
+ */
73
+ function publicationURI(did, origin) {
74
+ return `at://${did}/${PUBLICATION}/${createRkey(origin)}`
75
+ }
76
+
77
+ /**
78
+ * A `site.standard.document` record for the page, or the list of what it
79
+ * is missing. `title`, `publishedAt` and `site` are required by the
80
+ * lexicon; a page without a date cannot be a document. `site` is the
81
+ * publication's AT-URI; `origin` is what the page's key is made from.
82
+ * @param {{ metadata: Record<string, any> }} target
83
+ * @param {string | undefined} origin
84
+ * @param {string} [site] - the publication's AT-URI
85
+ * @returns {{ record: Record<string, unknown> } | { missing: string[] }}
86
+ */
87
+ function documentRecord(target, origin, site = origin) {
88
+ const { title, date, prettyURL, description, tags, hastAbstract } = target.metadata
89
+ const published = date ? new Date(date) : undefined
90
+ const validDate = published && !Number.isNaN(published.getTime())
91
+
92
+ const missing = [
93
+ ...(origin ? [] : ["domain"]),
94
+ ...(title ? [] : ["title"]),
95
+ ...(validDate ? [] : ["date"])
96
+ ]
97
+ if (missing.length) return { missing }
98
+
99
+ const textContent = hastAbstract ? toText(hastAbstract).trim() : ""
100
+
101
+ const record = {
102
+ $type: DOCUMENT,
103
+ site,
104
+ path: prettyURL,
105
+ title,
106
+ publishedAt: published.toISOString(),
107
+ ...(description ? { description } : {}),
108
+ ...(Array.isArray(tags) && tags.length ? { tags } : {}),
109
+ ...(textContent ? { textContent } : {})
110
+ }
111
+ return { record }
112
+ }
113
+
114
+ /**
115
+ * The page's AT-URI and record, or why it has none. `skip` is silent (the
116
+ * page is not meant to be published). `incomplete` is a page under a
117
+ * lexicon that lacks a property the lexicon requires; it is not a record,
118
+ * by design, and is only mentioned under `verbose`. `warning` is a
119
+ * misconfiguration: a lexicon vowel cannot build.
120
+ * @param {{ path: string, source?: string, metadata: Record<string, any> }} target
121
+ * @param {{ did: string, domain: unknown, settings: any }} context
122
+ * @returns {{ atUri: string, record: Record<string, unknown> } | { skip: true } | { incomplete: string } | { warning: string }}
123
+ */
124
+ function pageRecord(target, { did, domain, settings }) {
125
+ const lexicon = lexiconFor(target, settings)
126
+ if (!lexicon) return { skip: true }
127
+ if (!SUPPORTED.includes(lexicon)) {
128
+ return { warning: `lexicon "${lexicon}" is not one vowel can build a record for (supported: ${SUPPORTED.join(", ")})` }
129
+ }
130
+
131
+ const origin = siteOrigin(domain)
132
+ const site = origin ? publicationURI(did, origin) : undefined
133
+ const built = documentRecord(target, origin, site)
134
+ if ("missing" in built) {
135
+ return { incomplete: `not published to AT Protocol: a ${lexicon} record needs ${built.missing.join(", ")}` }
136
+ }
137
+
138
+ // The key is the page's canonical URL - origin and path - and frozen
139
+ // (rkey.js), whatever `site` holds.
140
+ const rkey = createRkey(new URL(String(built.record.path), origin).href)
141
+ return { atUri: `at://${did}/${lexicon}/${rkey}`, record: built.record }
142
+ }
143
+
144
+ export { DOCUMENT, PUBLICATION, SUPPORTED, lexiconFor, siteOrigin, publicationURI, documentRecord, pageRecord }
@@ -0,0 +1,54 @@
1
+ /**
2
+ * A page's record key, derived from its canonical URL: the site's origin
3
+ * and the page's published path, e.g. "https://example.com/blog/post".
4
+ *
5
+ * Deterministic, so the AT-URI is known on the first build, before any
6
+ * record exists, and the same page gets the same key on every machine. No
7
+ * ledger: the URL is the identity. The domain is part of it, so two sites
8
+ * publishing to one account never share a key, and a site that changes
9
+ * domain gets new keys: the documents now live somewhere else. The old
10
+ * records are *not* deleted by the next publish, because their `site` is
11
+ * the old domain and publish only deletes records whose `site` is this
12
+ * site's. A move that changes one page's URL, on the same domain, does
13
+ * delete that page's old record.
14
+ *
15
+ * `site.standard.document` declares `key: tid`, so the key must be shaped
16
+ * like a TID: 13 characters of base32-sortable, with the top bit clear.
17
+ * It is 63 bits of FNV-1a over the URL's UTF-8 bytes, which clears the
18
+ * top bit by construction. A real TID encodes a timestamp and this one
19
+ * does not, so keys do not sort by date. Nothing in site.standard relies
20
+ * on that.
21
+ *
22
+ * **The function is frozen once anything is published.** Changing the
23
+ * hash, its input or its encoding moves every page to a new key. The
24
+ * next publish would then delete and recreate every record.
25
+ */
26
+
27
+ const ALPHABET = "234567abcdefghijklmnopqrstuvwxyz"
28
+ const OFFSET = 0xcbf29ce484222325n
29
+ const PRIME = 0x100000001b3n
30
+ const MASK_64 = (1n << 64n) - 1n
31
+ const MASK_63 = (1n << 63n) - 1n
32
+ const LENGTH = 13
33
+
34
+ /** @param {string} text */
35
+ function fnv1a63(text) {
36
+ const bytes = Buffer.from(text, "utf8")
37
+ const hash = bytes.reduce((h, byte) => ((h ^ BigInt(byte)) * PRIME) & MASK_64, OFFSET)
38
+ return hash & MASK_63
39
+ }
40
+
41
+ /**
42
+ * @param {string} canonicalURL - e.g. "https://example.com/blog/post"
43
+ * @returns {string} a TID-shaped record key
44
+ */
45
+ function createRkey(canonicalURL) {
46
+ const hash = fnv1a63(canonicalURL)
47
+ const characters = Array.from({ length: LENGTH }, (_, index) => {
48
+ const shift = BigInt(5 * (LENGTH - 1 - index))
49
+ return ALPHABET[Number((hash >> shift) & 31n)]
50
+ })
51
+ return characters.join("")
52
+ }
53
+
54
+ export default createRkey