vowel 0.6.3 → 0.7.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -17
- package/bin.js +87 -30
- package/cli.js +15 -0
- package/config.js +102 -16
- package/extractDate.js +70 -6
- package/index.js +14 -5
- package/package.json +42 -7
- package/plugins/assets/index.js +65 -0
- package/plugins/atproto/index.js +132 -0
- package/plugins/atproto/password.js +71 -0
- package/plugins/atproto/publish.js +406 -0
- package/plugins/atproto/record.js +144 -0
- package/plugins/atproto/rkey.js +54 -0
- package/plugins/deploy-cloudflare/index.js +92 -0
- package/plugins/fonts/index.js +18 -21
- package/plugins/html/assets/bundle-BeK1Hptv.css +21 -0
- package/plugins/html/bundle/assets/index-BeK1Hptv.css +21 -0
- package/plugins/html/bundle/index.js +21145 -0
- package/plugins/html/editor/EditButton.svelte +152 -0
- package/plugins/html/editor/Editor.svelte +51 -0
- package/plugins/html/editor/Overlays.svelte +6 -0
- package/plugins/html/editor/SettingsPanel.svelte +397 -0
- package/plugins/html/editor/Toolbar.svelte +644 -0
- package/plugins/html/editor/create-session.js +250 -0
- package/plugins/html/editor/directives.js +128 -0
- package/plugins/html/editor/frontmatter.js +97 -0
- package/plugins/html/editor/ingest.js +339 -0
- package/plugins/html/editor/main.js +94 -0
- package/plugins/html/editor/nodes/Alert.svelte +21 -0
- package/plugins/html/editor/nodes/Blockquote.svelte +9 -0
- package/plugins/html/editor/nodes/CodeBlock.svelte +19 -0
- package/plugins/html/editor/nodes/Embed.svelte +19 -0
- package/plugins/html/editor/nodes/Emphasis.svelte +11 -0
- package/plugins/html/editor/nodes/Heading.svelte +15 -0
- package/plugins/html/editor/nodes/Highlight.svelte +11 -0
- package/plugins/html/editor/nodes/Image.svelte +18 -0
- package/plugins/html/editor/nodes/InlineCode.svelte +11 -0
- package/plugins/html/editor/nodes/Link.svelte +17 -0
- package/plugins/html/editor/nodes/List.svelte +15 -0
- package/plugins/html/editor/nodes/ListItem.svelte +9 -0
- package/plugins/html/editor/nodes/Page.svelte +21 -0
- package/plugins/html/editor/nodes/Paragraph.svelte +9 -0
- package/plugins/html/editor/nodes/Strikethrough.svelte +11 -0
- package/plugins/html/editor/nodes/Strong.svelte +11 -0
- package/plugins/html/editor/nodes/ThematicBreak.svelte +9 -0
- package/plugins/html/editor/nodes/index.js +18 -0
- package/plugins/html/editor/rollup.config.js +35 -0
- package/plugins/html/editor/serialize.js +157 -0
- package/plugins/html/editor/settings-file.js +83 -0
- package/plugins/html/editor/socket.js +202 -0
- package/plugins/html/editor/source.js +43 -0
- package/plugins/html/image.js +104 -0
- package/plugins/html/index.js +760 -0
- package/plugins/html/partials.js +299 -0
- package/plugins/html/socialLinks.js +67 -0
- package/plugins/html/writeRules.js +377 -0
- package/plugins/icons/index.js +2 -6
- package/plugins/images/index.js +78 -55
- package/plugins/markdown/brandColors.js +10 -0
- package/plugins/markdown/colorScheme.js +64 -0
- package/plugins/markdown/emoji.js +30 -0
- package/plugins/markdown/frontmatter.js +253 -0
- package/plugins/markdown/index.js +287 -1274
- package/plugins/markdown/links.js +97 -0
- package/plugins/markdown/metadata.js +308 -0
- package/plugins/markdown/readRules.js +143 -0
- package/plugins/markdown/typography.js +109 -0
- package/plugins/markdown/visitor.js +143 -0
- package/plugins/publish/index.js +34 -0
- package/plugins/robots/agents.js +79 -0
- package/plugins/robots/index.js +30 -10
- package/plugins/robots/robots.js +48 -0
- package/plugins/styles/index.js +111 -26
- package/plugins/styles/theme.js +116 -0
- package/plugins/urls/index.js +78 -0
- package/plugins/urls/urlMetadata.js +246 -0
- package/plugins/vectors/index.js +5 -9
- package/plugins/xml/entryContent.js +103 -0
- package/plugins/xml/index.js +123 -50
- package/secretPaths.js +170 -0
- package/stylesheets/DefaultStyles.css +367 -437
- package/stylesheets/ResetStyles.css +2 -38
- package/stylesheets/SyntaxHighlightingStyles.css +269 -0
- package/stylesheets/TypographyStyles.css +386 -128
- package/systemPaths.js +48 -0
- package/utils.js +56 -5
- package/.github/ISSUE_TEMPLATE/bug_report.md +0 -73
- package/.github/ISSUE_TEMPLATE/config.yml +0 -7
- package/docs-source/.cache.json +0 -9
- package/docs-source/.obsidian/app.json +0 -3
- package/docs-source/.obsidian/appearance.json +0 -3
- package/docs-source/.obsidian/core-plugins-migration.json +0 -30
- package/docs-source/.obsidian/core-plugins.json +0 -20
- package/docs-source/.obsidian/workspace.json +0 -168
- package/docs-source/.stackblitzrc +0 -3
- package/docs-source/.votive.db +0 -0
- package/docs-source/about.md +0 -3
- package/docs-source/blog/home.md +0 -5
- package/docs-source/blog/url-ui.md +0 -21
- package/docs-source/blog.md +0 -5
- package/docs-source/docs/.votive.db +0 -0
- package/docs-source/docs/deploy.md +0 -67
- package/docs-source/docs/file-structure.md +0 -31
- package/docs-source/docs/folder-settings.md +0 -23
- package/docs-source/docs/home.md +0 -55
- package/docs-source/docs/images.md +0 -10
- package/docs-source/docs/items.md +0 -13
- package/docs-source/docs/pages.md +0 -141
- package/docs-source/docs/settings.md +0 -4
- package/docs-source/docs/styling.md +0 -34
- package/docs-source/docs/taxonomies.md +0 -37
- package/docs-source/features/cards.md +0 -7
- package/docs-source/features/editing.md +0 -7
- package/docs-source/features/emoji.md +0 -7
- package/docs-source/features/frontmatter.md +0 -7
- package/docs-source/features/lists.md +0 -7
- package/docs-source/features/navigation.md +0 -7
- package/docs-source/features/rich-previews.md +0 -7
- package/docs-source/features/robots.md +0 -7
- package/docs-source/features/rss.md +0 -7
- package/docs-source/features/sitemap.md +0 -7
- package/docs-source/features/speed.md +0 -7
- package/docs-source/features/static.md +0 -7
- package/docs-source/features/taxonomies.md +0 -7
- package/docs-source/home.md +0 -42
- package/docs-source/output/about.html +0 -1
- package/docs-source/output/blog/url-ui.html +0 -8
- package/docs-source/output/blog.html +0 -1
- package/docs-source/output/default.css +0 -1
- package/docs-source/output/docs/deploy.html +0 -34
- package/docs-source/output/docs/file-structure.html +0 -24
- package/docs-source/output/docs/folder-settings.html +0 -20
- package/docs-source/output/docs/images.html +0 -2
- package/docs-source/output/docs/items.html +0 -6
- package/docs-source/output/docs/pages.html +0 -101
- package/docs-source/output/docs/styling.html +0 -18
- package/docs-source/output/docs/taxonomies.html +0 -20
- package/docs-source/output/docs.html +0 -28
- package/docs-source/output/features/cards.html +0 -1
- package/docs-source/output/features/editing.html +0 -1
- package/docs-source/output/features/emoji.html +0 -1
- package/docs-source/output/features/frontmatter.html +0 -1
- package/docs-source/output/features/lists.html +0 -1
- package/docs-source/output/features/navigation.html +0 -1
- package/docs-source/output/features/rich-previews.html +0 -1
- package/docs-source/output/features/robots.html +0 -1
- package/docs-source/output/features/rss.html +0 -1
- package/docs-source/output/features/sitemap.html +0 -1
- package/docs-source/output/features/speed.html +0 -1
- package/docs-source/output/features/static.html +0 -1
- package/docs-source/output/features/taxonomies.html +0 -1
- package/docs-source/output/features.html +0 -1
- package/docs-source/output/feed.xml +0 -1
- package/docs-source/output/index.html +0 -21
- package/docs-source/output/reset.css +0 -1
- package/docs-source/output/roadmap.html +0 -87
- package/docs-source/output/robots.txt +0 -14
- package/docs-source/output/sitemap.xml +0 -16
- package/docs-source/output/typography.css +0 -1
- package/docs-source/roadmap.md +0 -98
- package/docs-source/settings.md +0 -13
- package/getMetadata.js +0 -41
- package/jsconfig.json +0 -10
- package/regex.js +0 -36
- package/stylesheets/brand-colors.css +0 -119
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
import { createHash } from "node:crypto"
|
|
2
|
+
import { mkdtemp, rm, readFile } from "node:fs/promises"
|
|
3
|
+
import { tmpdir } from "node:os"
|
|
4
|
+
import path from "node:path"
|
|
5
|
+
import sharp from "sharp"
|
|
6
|
+
import votive from "votive"
|
|
7
|
+
import { systemDirectoryFor } from "../../systemPaths.js"
|
|
8
|
+
import { DOCUMENT, PUBLICATION, siteOrigin, publicationURI } from "./record.js"
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* `vowel --command atproto-publish`: make the account's records match the
|
|
12
|
+
* site.
|
|
13
|
+
*
|
|
14
|
+
* The PDS is the ledger. The command builds the site, collects the record
|
|
15
|
+
* each page's transform stored (`metadata.atprotoRecord`), lists what the
|
|
16
|
+
* PDS holds, and sends the difference through `applyWrites`: a record the
|
|
17
|
+
* site has and the PDS does not is created, one that differs is updated,
|
|
18
|
+
* and one the PDS has for this site and the site no longer produces is
|
|
19
|
+
* deleted. Nothing is remembered between runs, so clearing a cache
|
|
20
|
+
* cannot cause a resend, a revert publishes, and a coworker sees the
|
|
21
|
+
* same state.
|
|
22
|
+
*
|
|
23
|
+
* **Only this site's records are ever deleted.** `listRecords` returns
|
|
24
|
+
* every record in the collection, including ones another site or app
|
|
25
|
+
* published to the same account. A record is this site's when its `site`
|
|
26
|
+
* is this site's origin. A record at one of our keys with another `site`
|
|
27
|
+
* is a collision; it is reported and left alone.
|
|
28
|
+
*
|
|
29
|
+
* The build uses a throwaway database, so every page is read and every
|
|
30
|
+
* transform runs against current settings. A transform only re-runs when
|
|
31
|
+
* its source is re-read (tasks/1-proposed/transform-reruns-on-stale.md),
|
|
32
|
+
* so a warm database could hold records built from old settings.
|
|
33
|
+
*
|
|
34
|
+
* **The publication goes first.** One `site.standard.publication` per
|
|
35
|
+
* site, at the key record.js derives from the origin: `url` from
|
|
36
|
+
* `domain:`, `name` from the root `name:` (else the homepage's title),
|
|
37
|
+
* and `description` and `icon` from the root's `atproto_publication:`.
|
|
38
|
+
* The icon defaults to `logo:`. It is rendered to a 512px square PNG and
|
|
39
|
+
* its blob reference - CID, type, size - computed here, so an unchanged
|
|
40
|
+
* icon compares equal and is never uploaded again; a changed one is
|
|
41
|
+
* uploaded just before the write that names it. The publication is
|
|
42
|
+
* created and updated, never deleted.
|
|
43
|
+
*
|
|
44
|
+
* The app password comes from `ATPROTO_APP_PASSWORD`, or from a file
|
|
45
|
+
* named `atproto-app-password` in the project's system directory, never
|
|
46
|
+
* from the project. Payload: `{"dryRun": true}` reports what would change
|
|
47
|
+
* and sends nothing, and does not need the password. `{"optional": true}`
|
|
48
|
+
* makes a site with no `atproto_did` a skip rather than an error, for
|
|
49
|
+
* the `publish` command, which runs this after every deploy.
|
|
50
|
+
*/
|
|
51
|
+
|
|
52
|
+
const PASSWORD_FILE = "atproto-app-password"
|
|
53
|
+
const PAGE_SIZE = 100
|
|
54
|
+
const BATCH_SIZE = 100
|
|
55
|
+
const TIMEOUT = 30_000
|
|
56
|
+
const ICON_SIZE = 512
|
|
57
|
+
// The lexicon's maxSize for `icon`.
|
|
58
|
+
const ICON_MAX_BYTES = 1_000_000
|
|
59
|
+
const BASE32 = "abcdefghijklmnopqrstuvwxyz234567"
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* RFC 4648 base32, lowercase, unpadded: the multibase `b` alphabet.
|
|
63
|
+
* @param {Uint8Array} bytes
|
|
64
|
+
*/
|
|
65
|
+
function base32(bytes) {
|
|
66
|
+
const bits = [...bytes].map(byte => byte.toString(2).padStart(8, "0")).join("")
|
|
67
|
+
const padded = bits.padEnd(Math.ceil(bits.length / 5) * 5, "0")
|
|
68
|
+
return padded.match(/.{5}/g).map(chunk => BASE32[parseInt(chunk, 2)]).join("")
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* The CID a PDS gives a blob: CIDv1, raw codec (0x55), sha2-256.
|
|
73
|
+
* @param {Uint8Array} bytes
|
|
74
|
+
*/
|
|
75
|
+
function blobCID(bytes) {
|
|
76
|
+
const digest = createHash("sha256").update(bytes).digest()
|
|
77
|
+
return "b" + base32(Buffer.concat([Buffer.from([0x01, 0x55, 0x12, 0x20]), digest]))
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* The publication's icon: a 512px square PNG, the image contained in it
|
|
82
|
+
* on transparency, and the blob reference a PDS will give it. An SVG is
|
|
83
|
+
* rasterised at a density high enough to stay sharp at that size.
|
|
84
|
+
* @param {Buffer} source
|
|
85
|
+
* @param {string} iconPath - for the error message
|
|
86
|
+
*/
|
|
87
|
+
async function iconBlob(source, iconPath) {
|
|
88
|
+
const bytes = await sharp(source, { density: 300 })
|
|
89
|
+
.resize(ICON_SIZE, ICON_SIZE, { fit: "contain", background: { r: 0, g: 0, b: 0, alpha: 0 } })
|
|
90
|
+
.png()
|
|
91
|
+
.toBuffer()
|
|
92
|
+
if (bytes.length > ICON_MAX_BYTES) {
|
|
93
|
+
throw new Error(`atproto: the publication icon ${iconPath} is ${bytes.length} bytes as a ${ICON_SIZE}px PNG; the limit is ${ICON_MAX_BYTES}`)
|
|
94
|
+
}
|
|
95
|
+
const ref = { $type: "blob", ref: { $link: blobCID(bytes) }, mimeType: "image/png", size: bytes.length }
|
|
96
|
+
return { bytes, ref }
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* A value with its object keys sorted, so two records compare by content
|
|
101
|
+
* and not by key order.
|
|
102
|
+
* @param {unknown} value
|
|
103
|
+
* @returns {unknown}
|
|
104
|
+
*/
|
|
105
|
+
function canonical(value) {
|
|
106
|
+
if (Array.isArray(value)) return value.map(canonical)
|
|
107
|
+
if (!value || typeof value !== "object") return value
|
|
108
|
+
const keys = Object.keys(value).sort()
|
|
109
|
+
return Object.fromEntries(keys.map(key => [key, canonical(value[key])]))
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** @param {unknown} a @param {unknown} b */
|
|
113
|
+
function sameRecord(a, b) {
|
|
114
|
+
return JSON.stringify(canonical(a)) === JSON.stringify(canonical(b))
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* One XRPC call. An error response throws with the PDS's own error name
|
|
119
|
+
* and message, which is the only useful part of a failed write.
|
|
120
|
+
* @param {string} pds
|
|
121
|
+
* @param {string} nsid
|
|
122
|
+
* `bytes` is a raw body (uploadBlob) of type `mimeType`; `body` is JSON.
|
|
123
|
+
* @param {{ query?: Record<string, string>, body?: unknown, bytes?: Uint8Array, mimeType?: string, token?: string }} [options]
|
|
124
|
+
*/
|
|
125
|
+
async function xrpc(pds, nsid, { query, body, bytes, mimeType, token } = {}) {
|
|
126
|
+
const url = new URL(`/xrpc/${nsid}`, pds)
|
|
127
|
+
const params = new URLSearchParams(query ?? {})
|
|
128
|
+
url.search = params.toString()
|
|
129
|
+
const payload = bytes ?? (body === undefined ? undefined : JSON.stringify(body))
|
|
130
|
+
const headers = {
|
|
131
|
+
...(bytes ? { "content-type": mimeType } : body === undefined ? {} : { "content-type": "application/json" }),
|
|
132
|
+
...(token ? { authorization: `Bearer ${token}` } : {})
|
|
133
|
+
}
|
|
134
|
+
const response = await fetch(url, {
|
|
135
|
+
method: payload === undefined ? "GET" : "POST",
|
|
136
|
+
headers,
|
|
137
|
+
body: payload,
|
|
138
|
+
signal: AbortSignal.timeout(TIMEOUT)
|
|
139
|
+
})
|
|
140
|
+
const text = await response.text()
|
|
141
|
+
const json = text ? JSON.parse(text) : {}
|
|
142
|
+
if (response.ok) return json
|
|
143
|
+
throw new Error(`${nsid}: ${response.status} ${json.error ?? ""} ${json.message ?? ""}`.trim())
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* The PDS that hosts `did`, from its DID document. `did:plc` is resolved
|
|
148
|
+
* through the PLC directory, `did:web` through its own host.
|
|
149
|
+
* @param {string} did
|
|
150
|
+
*/
|
|
151
|
+
async function resolvePDS(did) {
|
|
152
|
+
const documentURL = did.startsWith("did:plc:")
|
|
153
|
+
? `https://plc.directory/${did}`
|
|
154
|
+
: did.startsWith("did:web:")
|
|
155
|
+
? `https://${decodeURIComponent(did.slice("did:web:".length))}/.well-known/did.json`
|
|
156
|
+
: undefined
|
|
157
|
+
if (!documentURL) throw new Error(`atproto: can't resolve ${did}; set atproto_pds in settings.md`)
|
|
158
|
+
|
|
159
|
+
const response = await fetch(documentURL, { signal: AbortSignal.timeout(TIMEOUT) })
|
|
160
|
+
if (!response.ok) throw new Error(`atproto: resolving ${did} failed: ${response.status}`)
|
|
161
|
+
const document = await response.json()
|
|
162
|
+
const service = (document.service ?? []).find(entry => entry.id === "#atproto_pds" || entry.id === `${did}#atproto_pds`)
|
|
163
|
+
if (!service) throw new Error(`atproto: ${did} names no PDS`)
|
|
164
|
+
return service.serviceEndpoint
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Where the app password is kept: the project's system directory, never
|
|
169
|
+
* the project, so it is neither committed nor deployed.
|
|
170
|
+
* @param {string} sourceFolder
|
|
171
|
+
*/
|
|
172
|
+
function passwordFile(sourceFolder) {
|
|
173
|
+
return path.join(systemDirectoryFor(sourceFolder), PASSWORD_FILE)
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** @param {string} sourceFolder */
|
|
177
|
+
async function appPassword(sourceFolder) {
|
|
178
|
+
if (process.env.ATPROTO_APP_PASSWORD) return process.env.ATPROTO_APP_PASSWORD
|
|
179
|
+
const file = passwordFile(sourceFolder)
|
|
180
|
+
const text = await readFile(file, "utf-8").catch(() => "")
|
|
181
|
+
if (text.trim()) return text.trim()
|
|
182
|
+
throw new Error(`atproto: no app password. Set ATPROTO_APP_PASSWORD, or put one in ${file}`)
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Every record in `collection`, a page at a time.
|
|
187
|
+
* @param {string} pds
|
|
188
|
+
* @param {string} did
|
|
189
|
+
* @param {string} collection
|
|
190
|
+
* @param {string} [cursor]
|
|
191
|
+
* @returns {Promise<{ uri: string, value: any }[]>}
|
|
192
|
+
*/
|
|
193
|
+
async function listAll(pds, did, collection, cursor) {
|
|
194
|
+
const query = { repo: did, collection, limit: String(PAGE_SIZE), ...(cursor ? { cursor } : {}) }
|
|
195
|
+
const page = await xrpc(pds, "com.atproto.repo.listRecords", { query })
|
|
196
|
+
const records = page.records ?? []
|
|
197
|
+
if (!page.cursor || !records.length) return records
|
|
198
|
+
const rest = await listAll(pds, did, collection, page.cursor)
|
|
199
|
+
return [...records, ...rest]
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** The last segment of an AT-URI. @param {string} uri */
|
|
203
|
+
const rkeyOf = (uri) => uri.slice(uri.lastIndexOf("/") + 1)
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* What the site should hold: every stored record, by rkey, the site's
|
|
207
|
+
* own settings, and the publication's name, description and icon. The
|
|
208
|
+
* icon is read from the build's output - it is a URL path from the root
|
|
209
|
+
* (frontmatter.js resolves it), which is where the output has it. Built
|
|
210
|
+
* into a scratch folder that is removed afterwards.
|
|
211
|
+
* @param {import("votive").VotiveConfig} config
|
|
212
|
+
*/
|
|
213
|
+
async function collectRecords(config) {
|
|
214
|
+
const scratch = await mkdtemp(path.join(tmpdir(), "vowel-atproto-"))
|
|
215
|
+
try {
|
|
216
|
+
const site = await votive({
|
|
217
|
+
...config,
|
|
218
|
+
verbose: false,
|
|
219
|
+
databasePath: path.join(scratch, ".votive.db"),
|
|
220
|
+
targetFolder: path.join(scratch, "output")
|
|
221
|
+
})
|
|
222
|
+
try {
|
|
223
|
+
await (await site.build({ defer: false })).deferred
|
|
224
|
+
const root = site.database.setting.getByFolder("")
|
|
225
|
+
const did = root.lastNonNull("fm_atproto_did")
|
|
226
|
+
const pds = root.lastNonNull("fm_atproto_pds")
|
|
227
|
+
const domain = root.lastNonNull("fm_domain")
|
|
228
|
+
const targets = site.database.target.getAll()
|
|
229
|
+
const pages = targets.filter(target => "atUri" in target.metadata && "atprotoRecord" in target.metadata)
|
|
230
|
+
const records = pages.map(target => ({ uri: target.metadata.atUri, value: target.metadata.atprotoRecord }))
|
|
231
|
+
|
|
232
|
+
const settings = root.lastNonNull("fm_atproto_publication") ?? {}
|
|
233
|
+
const home = site.database.target.get("index.html")
|
|
234
|
+
const name = root.raw("fm_name")?.[0]?.at(-1) ?? (home?.source ? home.metadata.title : undefined)
|
|
235
|
+
const iconPath = settings.icon ?? root.lastNonNull("logo")
|
|
236
|
+
const icon = iconPath ? await readIcon(path.join(scratch, "output"), iconPath) : undefined
|
|
237
|
+
const publication = { name, description: settings.description, iconPath, icon }
|
|
238
|
+
return { did, pds, domain, records, publication }
|
|
239
|
+
} finally {
|
|
240
|
+
await site.close()
|
|
241
|
+
}
|
|
242
|
+
} finally {
|
|
243
|
+
await rm(scratch, { recursive: true, force: true })
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* The icon's bytes from the build output. A URL is not fetched: the icon
|
|
249
|
+
* has to be a file in the project.
|
|
250
|
+
* @param {string} output
|
|
251
|
+
* @param {string} iconPath - a URL path from the root, e.g. "/logo.svg"
|
|
252
|
+
*/
|
|
253
|
+
async function readIcon(output, iconPath) {
|
|
254
|
+
if (/^[a-z]+:/i.test(iconPath)) {
|
|
255
|
+
throw new Error(`atproto: the publication icon must be a file in the project, not ${iconPath}`)
|
|
256
|
+
}
|
|
257
|
+
const file = path.join(output, ...iconPath.split("/").filter(Boolean))
|
|
258
|
+
return readFile(file).catch(() => {
|
|
259
|
+
throw new Error(`atproto: the publication icon ${iconPath} is not in the built site`)
|
|
260
|
+
})
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* The publication's write: create, update or unchanged, against what the
|
|
265
|
+
* account holds at its key.
|
|
266
|
+
* @param {{ uri: string, value: any }[]} remote - the account's publications
|
|
267
|
+
* @param {string} rkey
|
|
268
|
+
* @param {Record<string, unknown>} value
|
|
269
|
+
*/
|
|
270
|
+
function planPublication(remote, rkey, value) {
|
|
271
|
+
const existing = remote.find(record => rkeyOf(record.uri) === rkey)
|
|
272
|
+
if (!existing) return { action: "create", rkey, value }
|
|
273
|
+
if (sameRecord(existing.value, value)) return { action: "unchanged", rkey }
|
|
274
|
+
return { action: "update", rkey, value }
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* The writes that make `remote` match `local`, plus the collisions.
|
|
279
|
+
* @param {{ uri: string, value: any }[]} local
|
|
280
|
+
* @param {{ uri: string, value: any }[]} remote
|
|
281
|
+
* @param {Set<string>} sites - the `site` values that are this site's
|
|
282
|
+
*/
|
|
283
|
+
function planWrites(local, remote, sites) {
|
|
284
|
+
const remoteByKey = new Map(remote.map(record => [rkeyOf(record.uri), record]))
|
|
285
|
+
const localKeys = new Set(local.map(record => rkeyOf(record.uri)))
|
|
286
|
+
|
|
287
|
+
const compared = local.map(record => {
|
|
288
|
+
const rkey = rkeyOf(record.uri)
|
|
289
|
+
const existing = remoteByKey.get(rkey)
|
|
290
|
+
if (!existing) return { action: "create", rkey, value: record.value }
|
|
291
|
+
if (!sites.has(existing.value?.site)) return { action: "collision", rkey, site: existing.value?.site }
|
|
292
|
+
if (sameRecord(existing.value, record.value)) return { action: "unchanged", rkey }
|
|
293
|
+
return { action: "update", rkey, value: record.value }
|
|
294
|
+
})
|
|
295
|
+
|
|
296
|
+
const deletions = remote
|
|
297
|
+
.filter(record => sites.has(record.value?.site) && !localKeys.has(rkeyOf(record.uri)))
|
|
298
|
+
.map(record => ({ action: "delete", rkey: rkeyOf(record.uri) }))
|
|
299
|
+
|
|
300
|
+
return [...compared, ...deletions]
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** @param {{ action: string, rkey: string, value?: unknown, collection?: string }} write */
|
|
304
|
+
function applyWrite({ action, rkey, value, collection = DOCUMENT }) {
|
|
305
|
+
const base = { $type: `com.atproto.repo.applyWrites#${action}`, collection, rkey }
|
|
306
|
+
return action === "delete" ? base : { ...base, value }
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* @param {{ dryRun?: boolean, optional?: boolean } | undefined} payload
|
|
311
|
+
* @param {{ config: import("votive").VotiveConfig, notify: (message: object) => void }} context
|
|
312
|
+
*/
|
|
313
|
+
async function publishToAtproto(payload, { config, notify }) {
|
|
314
|
+
const dryRun = Boolean(payload?.dryRun)
|
|
315
|
+
|
|
316
|
+
notify({ status: "progress", message: "Building..." })
|
|
317
|
+
const { did, pds: configuredPDS, domain, records, publication } = await collectRecords(config)
|
|
318
|
+
|
|
319
|
+
if ((typeof did !== "string" || !did) && payload?.optional) {
|
|
320
|
+
notify({ status: "done", message: "No atproto_did in settings.md; nothing to publish to AT Protocol" })
|
|
321
|
+
return { skipped: true }
|
|
322
|
+
}
|
|
323
|
+
if (typeof did !== "string" || !did) throw new Error("atproto: set atproto_did in the root settings.md")
|
|
324
|
+
const origin = siteOrigin(domain)
|
|
325
|
+
if (!origin) throw new Error("atproto: set domain in the root settings.md; every record carries the site's url")
|
|
326
|
+
|
|
327
|
+
const accounts = new Set(records.map(record => record.uri.split("/")[2]))
|
|
328
|
+
const others = [...accounts].filter(account => account !== did)
|
|
329
|
+
if (others.length) throw new Error(`atproto: pages name ${others.join(", ")}; one site publishes to one account (${did})`)
|
|
330
|
+
|
|
331
|
+
if (typeof publication.name !== "string" || !publication.name) {
|
|
332
|
+
throw new Error("atproto: set name in the root settings.md; the publication needs one")
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
const icon = publication.icon ? await iconBlob(publication.icon, publication.iconPath) : undefined
|
|
336
|
+
const publicationRecord = {
|
|
337
|
+
$type: PUBLICATION,
|
|
338
|
+
url: origin,
|
|
339
|
+
name: publication.name,
|
|
340
|
+
...(typeof publication.description === "string" && publication.description ? { description: publication.description } : {}),
|
|
341
|
+
...(icon ? { icon: icon.ref } : {})
|
|
342
|
+
}
|
|
343
|
+
const publicationURL = publicationURI(did, origin)
|
|
344
|
+
|
|
345
|
+
// A document is this site's when its `site` is the publication, or the
|
|
346
|
+
// origin a loose document carried before there was one.
|
|
347
|
+
const sites = new Set([origin, publicationURL, ...records.map(record => record.value.site)])
|
|
348
|
+
const pds = typeof configuredPDS === "string" && configuredPDS ? configuredPDS : await resolvePDS(did)
|
|
349
|
+
|
|
350
|
+
notify({ status: "progress", message: `Comparing ${records.length} records with ${pds}...` })
|
|
351
|
+
const remote = await listAll(pds, did, DOCUMENT)
|
|
352
|
+
const remotePublications = await listAll(pds, did, PUBLICATION)
|
|
353
|
+
const publicationWrite = planPublication(remotePublications, rkeyOf(publicationURL), publicationRecord)
|
|
354
|
+
const plan = planWrites(records, remote, sites)
|
|
355
|
+
|
|
356
|
+
const count = (action) => plan.filter(write => write.action === action).length
|
|
357
|
+
const summary = {
|
|
358
|
+
created: count("create"),
|
|
359
|
+
updated: count("update"),
|
|
360
|
+
deleted: count("delete"),
|
|
361
|
+
unchanged: count("unchanged"),
|
|
362
|
+
collisions: plan.filter(write => write.action === "collision").map(({ rkey, site }) => ({ rkey, site })),
|
|
363
|
+
publication: publicationWrite.action,
|
|
364
|
+
dryRun
|
|
365
|
+
}
|
|
366
|
+
// A command's config comes straight from the caller and may carry no
|
|
367
|
+
// logger; votive() adds one only to the config its hooks receive.
|
|
368
|
+
const log = config.log ?? ((level, message) => console.error(message))
|
|
369
|
+
for (const { rkey, site } of summary.collisions) {
|
|
370
|
+
log("error", `atproto: ${DOCUMENT}/${rkey} belongs to ${site}, not ${origin}; left alone`)
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
const publicationWrites = publicationWrite.action === "unchanged" ? [] : [{ ...publicationWrite, collection: PUBLICATION }]
|
|
374
|
+
const documentWrites = plan.filter(write => ["create", "update", "delete"].includes(write.action))
|
|
375
|
+
const writes = [...publicationWrites, ...documentWrites].map(applyWrite)
|
|
376
|
+
if (dryRun || !writes.length) {
|
|
377
|
+
notify({ status: "done", message: `${dryRun ? "Would send" : "Nothing to send"}: ${JSON.stringify(summary)}` })
|
|
378
|
+
return summary
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
const password = await appPassword(config.sourceFolder)
|
|
382
|
+
const session = await xrpc(pds, "com.atproto.server.createSession", { body: { identifier: did, password } })
|
|
383
|
+
if (session.did !== did) throw new Error(`atproto: the app password is for ${session.did}, not ${did}`)
|
|
384
|
+
|
|
385
|
+
// The icon is uploaded only when the publication is written - which is
|
|
386
|
+
// when its reference changed, or the record is new.
|
|
387
|
+
if (icon && publicationWrites.length) {
|
|
388
|
+
notify({ status: "progress", message: "Uploading the publication icon..." })
|
|
389
|
+
const uploaded = await xrpc(pds, "com.atproto.repo.uploadBlob", { bytes: icon.bytes, mimeType: icon.ref.mimeType, token: session.accessJwt })
|
|
390
|
+
if (uploaded.blob?.ref?.$link !== icon.ref.ref.$link) {
|
|
391
|
+
throw new Error(`atproto: the PDS gave the icon ${uploaded.blob?.ref?.$link}, not the ${icon.ref.ref.$link} computed for it; nothing was written`)
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
const batches = Array.from({ length: Math.ceil(writes.length / BATCH_SIZE) }, (_, index) =>
|
|
396
|
+
writes.slice(index * BATCH_SIZE, (index + 1) * BATCH_SIZE))
|
|
397
|
+
for (const [index, batch] of batches.entries()) {
|
|
398
|
+
notify({ status: "progress", message: `Sending batch ${index + 1} of ${batches.length}...` })
|
|
399
|
+
await xrpc(pds, "com.atproto.repo.applyWrites", { body: { repo: did, writes: batch }, token: session.accessJwt })
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
notify({ status: "done", message: `Published: ${JSON.stringify(summary)}` })
|
|
403
|
+
return summary
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
export { publishToAtproto, planWrites, planPublication, sameRecord, resolvePDS, blobCID, xrpc, passwordFile }
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { toText } from "hast-util-to-text"
|
|
2
|
+
import { MARKER } from "../../secretPaths.js"
|
|
3
|
+
import createRkey from "./rkey.js"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Which pages become records, and what each record holds. No network:
|
|
7
|
+
* everything here is a function of the page and its settings, so it runs
|
|
8
|
+
* in the build, and the publish command only compares and sends.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const DOCUMENT = "site.standard.document"
|
|
12
|
+
const PUBLICATION = "site.standard.publication"
|
|
13
|
+
|
|
14
|
+
/** The lexicons vowel can build a record for. */
|
|
15
|
+
const SUPPORTED = [DOCUMENT]
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The page's lexicon and, for a secret page, whether it may use it.
|
|
19
|
+
*
|
|
20
|
+
* `atproto_lexicon:` cascades from settings.md, and a page's own
|
|
21
|
+
* frontmatter can set it too. A page under a secret segment (`##`) is published only when
|
|
22
|
+
* the lexicon was set *inside* its secret scope: in its own frontmatter,
|
|
23
|
+
* or in a settings.md at or below the secret folder. A lexicon inherited
|
|
24
|
+
* from above the secret would publish a hidden page to the network by
|
|
25
|
+
* accident.
|
|
26
|
+
*
|
|
27
|
+
* The settings chain is the page-as-folder chain of the routed path
|
|
28
|
+
* (`blog/post.html` reads `""`, `blog`, `blog/post`), so level `k` holds
|
|
29
|
+
* the first `k` segments. Routing replaces a segment with its hash but
|
|
30
|
+
* keeps the depth, so the secret segment at source index `s` is level
|
|
31
|
+
* `s + 1`, and every level from there down is inside it.
|
|
32
|
+
* @param {{ source?: string, metadata: Record<string, any> }} target
|
|
33
|
+
* @param {{ raw: (label: string) => (unknown[] | null)[] | undefined }} settings
|
|
34
|
+
* @returns {string | undefined}
|
|
35
|
+
*/
|
|
36
|
+
function lexiconFor(target, settings) {
|
|
37
|
+
const own = target.metadata.fm_atproto_lexicon
|
|
38
|
+
if (typeof own === "string" && own) return own
|
|
39
|
+
|
|
40
|
+
const levels = settings.raw("fm_atproto_lexicon") ?? []
|
|
41
|
+
const deepest = levels.findLastIndex(values => Array.isArray(values) && values.length > 0)
|
|
42
|
+
if (deepest === -1) return undefined
|
|
43
|
+
const lexicon = levels[deepest].at(-1)
|
|
44
|
+
if (typeof lexicon !== "string" || !lexicon) return undefined
|
|
45
|
+
|
|
46
|
+
const segments = (target.source ?? "").split(/[\\/]/)
|
|
47
|
+
const secret = segments.findIndex(segment => segment.includes(MARKER))
|
|
48
|
+
if (secret !== -1 && deepest < secret + 1) return undefined
|
|
49
|
+
return lexicon
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The site's origin from `domain:`, the same base the html plugin builds
|
|
54
|
+
* the canonical link from. No trailing slash, per the lexicon.
|
|
55
|
+
* @param {unknown} domain
|
|
56
|
+
*/
|
|
57
|
+
function siteOrigin(domain) {
|
|
58
|
+
if (typeof domain !== "string" || !domain) return undefined
|
|
59
|
+
const base = domain.startsWith("http") ? domain : `https://${domain}`
|
|
60
|
+
return new URL(base).origin
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* The site's publication: one per account and origin, its key derived
|
|
65
|
+
* from the origin the way a document's is from its URL, so its AT-URI is
|
|
66
|
+
* known before it exists. A document's `site` names it, and
|
|
67
|
+
* `/.well-known/site.standard.publication` serves it, which is how the
|
|
68
|
+
* domain claims it. The record itself is built by the publish command
|
|
69
|
+
* (publish.js), because its icon is an uploaded image.
|
|
70
|
+
* @param {string} did
|
|
71
|
+
* @param {string} origin
|
|
72
|
+
*/
|
|
73
|
+
function publicationURI(did, origin) {
|
|
74
|
+
return `at://${did}/${PUBLICATION}/${createRkey(origin)}`
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* A `site.standard.document` record for the page, or the list of what it
|
|
79
|
+
* is missing. `title`, `publishedAt` and `site` are required by the
|
|
80
|
+
* lexicon; a page without a date cannot be a document. `site` is the
|
|
81
|
+
* publication's AT-URI; `origin` is what the page's key is made from.
|
|
82
|
+
* @param {{ metadata: Record<string, any> }} target
|
|
83
|
+
* @param {string | undefined} origin
|
|
84
|
+
* @param {string} [site] - the publication's AT-URI
|
|
85
|
+
* @returns {{ record: Record<string, unknown> } | { missing: string[] }}
|
|
86
|
+
*/
|
|
87
|
+
function documentRecord(target, origin, site = origin) {
|
|
88
|
+
const { title, date, prettyURL, description, tags, hastAbstract } = target.metadata
|
|
89
|
+
const published = date ? new Date(date) : undefined
|
|
90
|
+
const validDate = published && !Number.isNaN(published.getTime())
|
|
91
|
+
|
|
92
|
+
const missing = [
|
|
93
|
+
...(origin ? [] : ["domain"]),
|
|
94
|
+
...(title ? [] : ["title"]),
|
|
95
|
+
...(validDate ? [] : ["date"])
|
|
96
|
+
]
|
|
97
|
+
if (missing.length) return { missing }
|
|
98
|
+
|
|
99
|
+
const textContent = hastAbstract ? toText(hastAbstract).trim() : ""
|
|
100
|
+
|
|
101
|
+
const record = {
|
|
102
|
+
$type: DOCUMENT,
|
|
103
|
+
site,
|
|
104
|
+
path: prettyURL,
|
|
105
|
+
title,
|
|
106
|
+
publishedAt: published.toISOString(),
|
|
107
|
+
...(description ? { description } : {}),
|
|
108
|
+
...(Array.isArray(tags) && tags.length ? { tags } : {}),
|
|
109
|
+
...(textContent ? { textContent } : {})
|
|
110
|
+
}
|
|
111
|
+
return { record }
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* The page's AT-URI and record, or why it has none. `skip` is silent (the
|
|
116
|
+
* page is not meant to be published). `incomplete` is a page under a
|
|
117
|
+
* lexicon that lacks a property the lexicon requires; it is not a record,
|
|
118
|
+
* by design, and is only mentioned under `verbose`. `warning` is a
|
|
119
|
+
* misconfiguration: a lexicon vowel cannot build.
|
|
120
|
+
* @param {{ path: string, source?: string, metadata: Record<string, any> }} target
|
|
121
|
+
* @param {{ did: string, domain: unknown, settings: any }} context
|
|
122
|
+
* @returns {{ atUri: string, record: Record<string, unknown> } | { skip: true } | { incomplete: string } | { warning: string }}
|
|
123
|
+
*/
|
|
124
|
+
function pageRecord(target, { did, domain, settings }) {
|
|
125
|
+
const lexicon = lexiconFor(target, settings)
|
|
126
|
+
if (!lexicon) return { skip: true }
|
|
127
|
+
if (!SUPPORTED.includes(lexicon)) {
|
|
128
|
+
return { warning: `lexicon "${lexicon}" is not one vowel can build a record for (supported: ${SUPPORTED.join(", ")})` }
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const origin = siteOrigin(domain)
|
|
132
|
+
const site = origin ? publicationURI(did, origin) : undefined
|
|
133
|
+
const built = documentRecord(target, origin, site)
|
|
134
|
+
if ("missing" in built) {
|
|
135
|
+
return { incomplete: `not published to AT Protocol: a ${lexicon} record needs ${built.missing.join(", ")}` }
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// The key is the page's canonical URL - origin and path - and frozen
|
|
139
|
+
// (rkey.js), whatever `site` holds.
|
|
140
|
+
const rkey = createRkey(new URL(String(built.record.path), origin).href)
|
|
141
|
+
return { atUri: `at://${did}/${lexicon}/${rkey}`, record: built.record }
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export { DOCUMENT, PUBLICATION, SUPPORTED, lexiconFor, siteOrigin, publicationURI, documentRecord, pageRecord }
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A page's record key, derived from its canonical URL: the site's origin
|
|
3
|
+
* and the page's published path, e.g. "https://example.com/blog/post".
|
|
4
|
+
*
|
|
5
|
+
* Deterministic, so the AT-URI is known on the first build, before any
|
|
6
|
+
* record exists, and the same page gets the same key on every machine. No
|
|
7
|
+
* ledger: the URL is the identity. The domain is part of it, so two sites
|
|
8
|
+
* publishing to one account never share a key, and a site that changes
|
|
9
|
+
* domain gets new keys: the documents now live somewhere else. The old
|
|
10
|
+
* records are *not* deleted by the next publish, because their `site` is
|
|
11
|
+
* the old domain and publish only deletes records whose `site` is this
|
|
12
|
+
* site's. A move that changes one page's URL, on the same domain, does
|
|
13
|
+
* delete that page's old record.
|
|
14
|
+
*
|
|
15
|
+
* `site.standard.document` declares `key: tid`, so the key must be shaped
|
|
16
|
+
* like a TID: 13 characters of base32-sortable, with the top bit clear.
|
|
17
|
+
* It is 63 bits of FNV-1a over the URL's UTF-8 bytes, which clears the
|
|
18
|
+
* top bit by construction. A real TID encodes a timestamp and this one
|
|
19
|
+
* does not, so keys do not sort by date. Nothing in site.standard relies
|
|
20
|
+
* on that.
|
|
21
|
+
*
|
|
22
|
+
* **The function is frozen once anything is published.** Changing the
|
|
23
|
+
* hash, its input or its encoding moves every page to a new key. The
|
|
24
|
+
* next publish would then delete and recreate every record.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
const ALPHABET = "234567abcdefghijklmnopqrstuvwxyz"
|
|
28
|
+
const OFFSET = 0xcbf29ce484222325n
|
|
29
|
+
const PRIME = 0x100000001b3n
|
|
30
|
+
const MASK_64 = (1n << 64n) - 1n
|
|
31
|
+
const MASK_63 = (1n << 63n) - 1n
|
|
32
|
+
const LENGTH = 13
|
|
33
|
+
|
|
34
|
+
/** @param {string} text */
|
|
35
|
+
function fnv1a63(text) {
|
|
36
|
+
const bytes = Buffer.from(text, "utf8")
|
|
37
|
+
const hash = bytes.reduce((h, byte) => ((h ^ BigInt(byte)) * PRIME) & MASK_64, OFFSET)
|
|
38
|
+
return hash & MASK_63
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* @param {string} canonicalURL - e.g. "https://example.com/blog/post"
|
|
43
|
+
* @returns {string} a TID-shaped record key
|
|
44
|
+
*/
|
|
45
|
+
function createRkey(canonicalURL) {
|
|
46
|
+
const hash = fnv1a63(canonicalURL)
|
|
47
|
+
const characters = Array.from({ length: LENGTH }, (_, index) => {
|
|
48
|
+
const shift = BigInt(5 * (LENGTH - 1 - index))
|
|
49
|
+
return ALPHABET[Number((hash >> shift) & 31n)]
|
|
50
|
+
})
|
|
51
|
+
return characters.join("")
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export default createRkey
|