uniweb 1.33.0 → 1.34.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/package.json +3 -3
- package/partials/agents.md +4 -4
- package/partials/guides/backend.md +1 -1
- package/partials/guides/data.md +28 -4
- package/partials/guides/ship.md +1 -0
- package/src/backend/asset-download.js +24 -6
- package/src/backend/library-types.js +56 -13
- package/src/backend/site-sync.js +13 -3
- package/src/commands/build.js +15 -12
- package/src/commands/publish.js +18 -120
- package/src/commands/pull.js +10 -1
- package/src/commands/push.js +0 -3
- package/src/commands/register.js +89 -13
- package/src/framework-index.json +5 -5
- package/src/utils/records-guard.js +7 -6
- package/templates/workspace/README.md.hbs +3 -0
- package/src/utils/schemaless-report.js +0 -30
- package/src/utils/site-data-upload.js +0 -310
|
@@ -1,310 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Static collection data — the passthrough lane.
|
|
3
|
-
*
|
|
4
|
-
* A site's **schema-less** queries have no entity model, so their compiled
|
|
5
|
-
* `dist/data/**` JSON is delivered as files rather than synced as entities.
|
|
6
|
-
* This plans and uploads that set: one plan call, one object per file, each
|
|
7
|
-
* PUT to the target the backend returns.
|
|
8
|
-
*
|
|
9
|
-
* 1. PLAN — POST {apiBase}/dev/site/data-uploads/{site} with the file list
|
|
10
|
-
* ({ path, content_type, size, sha256 }). Response carries
|
|
11
|
-
* `mode` ('presigned' | 'direct'), `expires_in`, `serve_base`,
|
|
12
|
-
* and one `uploads` entry per file ({ path, method, url, headers }).
|
|
13
|
-
* 2. UPLOAD — PUT each file's bytes to its URL. Order is irrelevant; there is
|
|
14
|
-
* no entry file and no server confirm step.
|
|
15
|
-
*
|
|
16
|
-
* ⭐ **`path` is the SERVING TAIL** (`data/articles.json`), not a bookkeeping
|
|
17
|
-
* key. The file lands where it is served — which is the whole point of this
|
|
18
|
-
* lane, and why it returns no map: nothing has to record where anything went.
|
|
19
|
-
*
|
|
20
|
-
* ## Why this is not the asset lane
|
|
21
|
-
*
|
|
22
|
-
* Assets are **content-addressed, and dedup across the sites of one backend**
|
|
23
|
-
* — so the relation from a site's path to an object is many-to-one and a serve
|
|
24
|
-
* path cannot be a property of the object. *("Global" names that dedup scope; it
|
|
25
|
-
* never meant across backends — see utils/asset-upload.js.)* This lane is **per-site and
|
|
26
|
-
* path-addressed**. Sending static data through the asset store is what forced
|
|
27
|
-
* the old "data ball": one bundled asset the backend had to fetch, parse and
|
|
28
|
-
* fan out, which is the only place these bytes ever transited it.
|
|
29
|
-
*
|
|
30
|
-
* ⇒ **The backend is a PASSTHROUGH here** — it brokers the authorization, not
|
|
31
|
-
* the bytes. Hosting static content is the hosting platform's mission where an
|
|
32
|
-
* edge exists; where there is none (edgeless), the backend serves them itself,
|
|
33
|
-
* and that is the posture working rather than a fallback.
|
|
34
|
-
*
|
|
35
|
-
* ⛔ **Do not compose paths onto `serve_base`.** It is `null` on the presigned
|
|
36
|
-
* arm (prod serving is the delivery tier's URL shape, not the backend's to
|
|
37
|
-
* mint) and the gateway's data route on the direct arm. Nothing in a build
|
|
38
|
-
* needs it — the site's fetcher asks for a site-relative `/data/{path}` at
|
|
39
|
-
* runtime — so it is returned for parity and for a caller that wants to verify
|
|
40
|
-
* an upload landed.
|
|
41
|
-
*
|
|
42
|
-
* Contract ratified with the backend lane, 2026-08-18.
|
|
43
|
-
* **Wired into `publish.js` and shipped in `uniweb` 0.25.3.** Both arms of the
|
|
44
|
-
* endpoint exist (presigned landed 2026-08-18, per Diego).
|
|
45
|
-
*
|
|
46
|
-
* ⚠️ **The presigned arm has never been exercised against a real backend from
|
|
47
|
-
* here.** The branch is one line and unit-tested both ways, but a stub is not a
|
|
48
|
-
* presigning deployment — three claims diverged that way in a single day while
|
|
49
|
-
* this lane was being built. A local `uniwebd` answers `direct`, so proving it
|
|
50
|
-
* needs a presigning deployment and one real publish. **Believed correct, not
|
|
51
|
-
* demonstrated.**
|
|
52
|
-
*/
|
|
53
|
-
|
|
54
|
-
import { WORKSPACE_HEADER, WorkspaceRefusedError } from '../backend/client.js'
|
|
55
|
-
import { createHash } from 'node:crypto'
|
|
56
|
-
import { humanBytes } from './bytes.js'
|
|
57
|
-
import { fetchWithRetry, isTransientStatus } from './fetch-retry.js'
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* Plan + upload a site's static collection data files.
|
|
61
|
-
*
|
|
62
|
-
* @param {object} opts
|
|
63
|
-
* @param {string} opts.apiBase - backend origin
|
|
64
|
-
* @param {string} opts.token - bearer, used on the direct arm only
|
|
65
|
-
* @param {string} opts.siteUuid - the site these files belong to (path segment)
|
|
66
|
-
* @param {string|null} [opts.workspace] - the workspace the requests name, `@handle`
|
|
67
|
-
* (`x-uniweb-workspace`) — on the plan and on every DIRECT PUT, never on a presigned
|
|
68
|
-
* one, which is self-authorizing and must carry nothing it was not signed with
|
|
69
|
-
* @param {string} [opts.workspaceSource] - where that workspace came from (`workspace.js`),
|
|
70
|
-
* so a refusal of it says how to choose another (`WorkspaceRefusedError`)
|
|
71
|
-
* @param {{ data: Record<string, unknown> }|null} opts.ball - source of the set:
|
|
72
|
-
* `{ "<relpath under dist/data>": <json> }`, media refs already rewritten
|
|
73
|
-
* @param {(m: string) => void} [opts.onProgress]
|
|
74
|
-
* @returns {Promise<{ mode: string, uploaded: string[], failed: Array<{path:string,status:number,detail?:string}>, serveBase: string|null }>}
|
|
75
|
-
* **No map** — by design. The files are at their serving paths.
|
|
76
|
-
*/
|
|
77
|
-
export async function uploadSiteData({
|
|
78
|
-
apiBase,
|
|
79
|
-
token,
|
|
80
|
-
siteUuid,
|
|
81
|
-
workspace = null,
|
|
82
|
-
workspaceSource = 'login',
|
|
83
|
-
ball,
|
|
84
|
-
onProgress = () => {}
|
|
85
|
-
}) {
|
|
86
|
-
// A plan made with the header mints BARE PUT urls: the PUT names its workspace with
|
|
87
|
-
// the header like every other request (backend, 2026-09-23).
|
|
88
|
-
const workspaceHeader = workspace ? { [WORKSPACE_HEADER]: workspace } : {}
|
|
89
|
-
// ⛔ An EMPTY set still posts a plan, and that is the whole point of this lane
|
|
90
|
-
// being a manifest rather than a stream of files.
|
|
91
|
-
//
|
|
92
|
-
// The backend reconciles a site's data usage against what this plan declares:
|
|
93
|
-
// whatever is not in the manifest is gone, so deleting a collection stops
|
|
94
|
-
// costing on the next publish. ⭐ A plan with zero files is a STATEMENT that
|
|
95
|
-
// there are none; the ABSENCE of a request says nothing at all. Returning
|
|
96
|
-
// early here — which this did until 2026-09-01 — made "the user deleted their
|
|
97
|
-
// last schema-less collection" unexpressible on the wire, so that site kept
|
|
98
|
-
// paying for bytes it no longer served until the whole site was deleted.
|
|
99
|
-
// Nothing looked wrong at either end: no error, no warning, just a request
|
|
100
|
-
// that was never sent.
|
|
101
|
-
//
|
|
102
|
-
// Agreed both sides in channel backend↔framework; the backend's plan
|
|
103
|
-
// route accepted an empty `files` array in the same exchange (it was a 400
|
|
104
|
-
// before, which is what made the omission look like the only option).
|
|
105
|
-
const entries = Object.entries(ball?.data || {})
|
|
106
|
-
|
|
107
|
-
// One plan for the whole set. The per-request file cap counts a plan, so
|
|
108
|
-
// splitting would evade it rather than respect it; if a set ever exceeds it,
|
|
109
|
-
// the refusal is the correct outcome and the cap is the thing to fix.
|
|
110
|
-
const files = entries.map(([relPath, value]) => {
|
|
111
|
-
const bytes = Buffer.from(JSON.stringify(value))
|
|
112
|
-
return {
|
|
113
|
-
// ⚠️ The tail RELATIVE TO THE DATA ROOT — no `data/` prefix.
|
|
114
|
-
//
|
|
115
|
-
// The plan's `serve_base` already ends in `/data/`
|
|
116
|
-
// (`/gateway/site/{site}/data/`), and the runtime asks for
|
|
117
|
-
// `{config.base}/data/{name}.json` (`DATA_URL_PREFIX`). Sending
|
|
118
|
-
// `data/articles.json` would store one level too deep and serve at
|
|
119
|
-
// `…/data/data/articles.json`, which the runtime never requests.
|
|
120
|
-
//
|
|
121
|
-
// ⛔ And the failure is INVISIBLE from here: the plan succeeds, the PUT
|
|
122
|
-
// succeeds, and only a visitor's fetch 404s. Confirmed against the
|
|
123
|
-
// shipped contract's own example (`data/articles.json` with a
|
|
124
|
-
// `/data/`-bearing `serve_base`), which supersedes an earlier
|
|
125
|
-
// parenthetical that showed the prefix.
|
|
126
|
-
path: relPath,
|
|
127
|
-
content_type: 'application/json',
|
|
128
|
-
size: bytes.length,
|
|
129
|
-
sha256: createHash('sha256').update(bytes).digest('hex'),
|
|
130
|
-
bytes
|
|
131
|
-
}
|
|
132
|
-
})
|
|
133
|
-
|
|
134
|
-
const origin = apiBase.replace(/\/$/, '')
|
|
135
|
-
// `{site}` is a PATH segment, not a body field — every sibling on this
|
|
136
|
-
// controller addresses the site that way (`content/push/{site}`,
|
|
137
|
-
// `publish/{site}`, `status/{site}`). The asset lane's body-borne owner is
|
|
138
|
-
// the wrong precedent: that lane is global, this one is per-site.
|
|
139
|
-
const planRes = await fetch(
|
|
140
|
-
`${origin}/dev/site/data-uploads/${encodeURIComponent(siteUuid)}`,
|
|
141
|
-
{
|
|
142
|
-
method: 'POST',
|
|
143
|
-
headers: {
|
|
144
|
-
'Content-Type': 'application/json',
|
|
145
|
-
Authorization: `Bearer ${token}`,
|
|
146
|
-
...workspaceHeader
|
|
147
|
-
},
|
|
148
|
-
body: JSON.stringify({
|
|
149
|
-
files: files.map(({ path, content_type, size, sha256 }) => ({
|
|
150
|
-
path,
|
|
151
|
-
content_type,
|
|
152
|
-
size,
|
|
153
|
-
sha256
|
|
154
|
-
}))
|
|
155
|
-
})
|
|
156
|
-
}
|
|
157
|
-
)
|
|
158
|
-
if (!planRes.ok) {
|
|
159
|
-
const body = await planRes.text().catch(() => '')
|
|
160
|
-
// The PARSED problem document has to survive the throw. Flattening it into
|
|
161
|
-
// the message is what left the other two doors unable to branch on `reason`
|
|
162
|
-
// and printing raw JSON at users; this lane was the last one still doing it.
|
|
163
|
-
// Callers read `err.problem`; `describeDataRefusal` turns it into lines.
|
|
164
|
-
let problem = null
|
|
165
|
-
if (body) {
|
|
166
|
-
try {
|
|
167
|
-
const parsed = JSON.parse(body)
|
|
168
|
-
if (parsed && typeof parsed === 'object') problem = parsed
|
|
169
|
-
} catch {
|
|
170
|
-
/* not a problem document — prose refusal, or an upstream error page */
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
// ⭐ The workspace the plan named is not one the account may open — said as every other
|
|
174
|
-
// site request says it (`WorkspaceRefusedError`, backend/client.js), never as a raw 403.
|
|
175
|
-
if (planRes.status === 403 && problem?.op === 'act_as_unit' && workspace) {
|
|
176
|
-
throw new WorkspaceRefusedError({ named: workspace, source: workspaceSource, origin })
|
|
177
|
-
}
|
|
178
|
-
const err = new Error(
|
|
179
|
-
`site data-uploads plan rejected: HTTP ${planRes.status}${body ? ` — ${body.slice(0, 300)}` : ''}`
|
|
180
|
-
)
|
|
181
|
-
err.status = planRes.status
|
|
182
|
-
err.problem = problem
|
|
183
|
-
throw err
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
const plan = await planRes.json()
|
|
187
|
-
const targets = new Map((plan.uploads || []).map((u) => [u.path, u]))
|
|
188
|
-
// The one mode-aware bit, same rule as the code and runtime lanes: a direct
|
|
189
|
-
// PUT is a bearer-authed backend route; a presigned URL is self-authorizing
|
|
190
|
-
// and must NOT carry a foreign bearer, which can break signature validation.
|
|
191
|
-
const authHeaders =
|
|
192
|
-
plan.mode === 'presigned'
|
|
193
|
-
? {}
|
|
194
|
-
: { Authorization: `Bearer ${token}`, ...workspaceHeader }
|
|
195
|
-
|
|
196
|
-
const uploaded = []
|
|
197
|
-
const failed = []
|
|
198
|
-
for (const f of files) {
|
|
199
|
-
const target = targets.get(f.path)
|
|
200
|
-
if (!target) {
|
|
201
|
-
// A file the plan did not answer for is unaddressable. Report it; never
|
|
202
|
-
// invent a location for it.
|
|
203
|
-
failed.push({
|
|
204
|
-
path: f.path,
|
|
205
|
-
status: 0,
|
|
206
|
-
detail: 'no upload target in plan'
|
|
207
|
-
})
|
|
208
|
-
continue
|
|
209
|
-
}
|
|
210
|
-
try {
|
|
211
|
-
// Relative on the direct arm, absolute on presigned — `new URL` resolves
|
|
212
|
-
// both against the origin.
|
|
213
|
-
// ⭐ **Retried.** A single connection-level failure used to fail the whole
|
|
214
|
-
// verb after every other file had already gone up. The PUT is idempotent
|
|
215
|
-
// here — same bytes, same target, verified by `x-uniweb-sha256` on receipt
|
|
216
|
-
// — so repeating it is safe by construction, which is why this opts in to
|
|
217
|
-
// `retryOnStatus` rather than inheriting it.
|
|
218
|
-
//
|
|
219
|
-
// ⚠️ 120s per attempt, not the helper's 30s default: this bounds the time
|
|
220
|
-
// to send a whole BODY, and aborting a legitimately slow upload only to
|
|
221
|
-
// retry it is worse than not timing out at all.
|
|
222
|
-
const res = await fetchWithRetry(new URL(target.url, origin), {
|
|
223
|
-
method: target.method || 'PUT',
|
|
224
|
-
headers: {
|
|
225
|
-
'Content-Type': 'application/json',
|
|
226
|
-
...(target.headers || {}),
|
|
227
|
-
...authHeaders
|
|
228
|
-
},
|
|
229
|
-
body: f.bytes
|
|
230
|
-
}, { timeoutMs: 120_000, retryOnStatus: isTransientStatus })
|
|
231
|
-
if (res.ok) {
|
|
232
|
-
uploaded.push(f.path)
|
|
233
|
-
onProgress(`${f.path}`)
|
|
234
|
-
} else {
|
|
235
|
-
failed.push({ path: f.path, status: res.status })
|
|
236
|
-
}
|
|
237
|
-
} catch (err) {
|
|
238
|
-
failed.push({ path: f.path, status: 0, detail: err.message })
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
return {
|
|
243
|
-
mode: plan.mode || 'direct',
|
|
244
|
-
uploaded,
|
|
245
|
-
failed,
|
|
246
|
-
serveBase: plan.serve_base || null
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
/**
|
|
251
|
-
* Turn a typed data-uploads refusal into user-facing lines, or null when there is
|
|
252
|
-
* no typed `reason` (⇒ fall through to the generic message, degrading rather than
|
|
253
|
-
* swallowing).
|
|
254
|
-
*
|
|
255
|
-
* ⭐ The THIRD door, and the last to get one. `/dev/assets` has
|
|
256
|
-
* `describeAssetRefusal`, a site create has `describeStorageRefusal`, and this
|
|
257
|
-
* lane threw prose with the JSON inlined until 2026-09-01 — which is the failure
|
|
258
|
-
* the other two describers exist to prevent, so leaving it was just an untreated
|
|
259
|
-
* instance of a solved problem.
|
|
260
|
-
*
|
|
261
|
-
* ⛔ Branch on `reason`, never the status: `507` alone cannot be told from any
|
|
262
|
-
* other `507` and carries none of the numbers, and `detail` is prose the backend
|
|
263
|
-
* may reword.
|
|
264
|
-
*
|
|
265
|
-
* ⚖️ **The advice DIVERGES from the asset lane's, and that divergence is the whole
|
|
266
|
-
* reason this is a separate function rather than a reused one.** On the asset lane
|
|
267
|
-
* removing an image frees nothing — freeing is entity-deletion-granular. Here the
|
|
268
|
-
* publish declares the COMPLETE set of schema-less data files every time, so
|
|
269
|
-
* dropping a collection and re-publishing is a real way to stop paying for it.
|
|
270
|
-
* Telling a data user "editing content frees nothing" would be false, and telling
|
|
271
|
-
* an asset user "just remove it" would be worse.
|
|
272
|
-
*
|
|
273
|
-
* @param {Error & { problem?: object|null }} err
|
|
274
|
-
* @returns {{ headline: string, notes: string[] } | null}
|
|
275
|
-
*/
|
|
276
|
-
export function describeDataRefusal(err) {
|
|
277
|
-
const p = err?.problem
|
|
278
|
-
const reason = p?.reason
|
|
279
|
-
if (typeof reason !== 'string') return null
|
|
280
|
-
|
|
281
|
-
if (reason === 'storage_quota_exceeded') {
|
|
282
|
-
const notes = []
|
|
283
|
-
const used = humanBytes(p.used_bytes)
|
|
284
|
-
const limit = humanBytes(p.limit_bytes)
|
|
285
|
-
const needed = humanBytes(p.needed_bytes)
|
|
286
|
-
if (used) notes.push(` Used: ${used}`)
|
|
287
|
-
if (limit) notes.push(` Limit: ${limit}`)
|
|
288
|
-
if (needed) notes.push(` This publish adds: ${needed}`)
|
|
289
|
-
notes.push(
|
|
290
|
-
'Every publish declares the full set of schema-less data files, so removing'
|
|
291
|
-
)
|
|
292
|
-
notes.push(
|
|
293
|
-
'a collection and re-publishing stops it counting. Deleting a site or entity'
|
|
294
|
-
)
|
|
295
|
-
notes.push('frees space too.')
|
|
296
|
-
return {
|
|
297
|
-
headline:
|
|
298
|
-
"Storage quota reached — the site owner's workspace cannot take on more record data.",
|
|
299
|
-
notes
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
// An unrecognised reason still beats a status dump: name it, and let the
|
|
304
|
-
// backend's own prose follow when it sent any.
|
|
305
|
-
const detail = typeof p.detail === 'string' ? p.detail : ''
|
|
306
|
-
return {
|
|
307
|
-
headline: `Record data upload refused by the backend (${reason}).`,
|
|
308
|
-
notes: detail ? [` ${detail}`] : []
|
|
309
|
-
}
|
|
310
|
-
}
|