pikelet-wasm 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +399 -2
- package/README.md +81 -257
- package/complete/builder.mjs +35 -5
- package/complete/format.mjs +6 -1
- package/complete/fusion.mjs +72 -0
- package/complete/index.d.ts +23 -0
- package/complete/index.mjs +580 -76
- package/complete/inline-transformer.mjs +97 -1
- package/complete/layer-append.mjs +246 -0
- package/complete/layer-chain.mjs +284 -0
- package/complete/layer-compact.mjs +319 -0
- package/complete/layer-encoder.mjs +223 -0
- package/complete/layer-locator.mjs +228 -0
- package/complete/layer-manifest.mjs +271 -0
- package/complete/layer-reader.mjs +759 -0
- package/complete/layer-rebase.mjs +218 -0
- package/complete/lexical.mjs +78 -8
- package/complete/retrieval-abstention.mjs +132 -28
- package/complete/sources.mjs +1 -1
- package/complete/tombstones.mjs +281 -0
- package/complete/wordpiece.mjs +22 -0
- package/dist/engine.js +1 -1
- package/dist/engine.scalar.js +1 -1
- package/dist/engine.scalar.wasm +0 -0
- package/dist/engine.wasm +0 -0
- package/package.json +65 -77
- package/{pikelet-artifact-common.js → src/artifact/pikelet-artifact-common.js} +1 -1
- package/{pikelet-artifact-range.js → src/artifact/pikelet-artifact-range.js} +8 -1
- package/{pikelet-artifact-sketch.js → src/artifact/pikelet-artifact-sketch.js} +97 -8
- package/{pikelet-artifact.d.ts → src/artifact/pikelet-artifact.d.ts} +1 -1
- package/{pikelet-core.js → src/core/pikelet-core.js} +6 -5
- package/{pikelet.js → src/pikelet.js} +7 -7
- package/{pikelet.node.mjs → src/pikelet.node.mjs} +7 -7
- package/{pikelet.web.mjs → src/pikelet.web.mjs} +8 -8
- package/{pikelet.workerd.mjs → src/pikelet.workerd.mjs} +8 -8
- /package/{pikelet-artifact.js → src/artifact/pikelet-artifact.js} +0 -0
- /package/{pikelet-errors.js → src/errors/pikelet-errors.js} +0 -0
- /package/{pikelet-loader.js → src/loader/pikelet-loader.js} +0 -0
- /package/{pikelet.d.ts → src/pikelet.d.ts} +0 -0
- /package/{pikelet.web.d.ts → src/pikelet.web.d.ts} +0 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
// Parent resolution — LAYERED_PROFILE.md 5.1.1.
|
|
2
|
+
//
|
|
3
|
+
// Identity verification (3.2) prevents substitution of bytes. It does NOT
|
|
4
|
+
// prevent a hostile manifest from directing a reader's fetches: a layer's
|
|
5
|
+
// `parent.locator` is publisher-supplied, and the publisher is the adversary
|
|
6
|
+
// for a distributed artifact. So a locator is confined before any fetch, and
|
|
7
|
+
// every redirect hop is confined again with the same rule, never a weaker one.
|
|
8
|
+
//
|
|
9
|
+
// The governing principle of 5.1.1, which shapes every rule below:
|
|
10
|
+
//
|
|
11
|
+
// "Normalization defects live in the disagreements between a validator and
|
|
12
|
+
// the server that finally interprets the path, so the reader refuses every
|
|
13
|
+
// form on which the two could disagree rather than picking one reading."
|
|
14
|
+
//
|
|
15
|
+
// That is why this module rejects rather than normalizes: an encoded separator
|
|
16
|
+
// (%2f), an encoded or literal backslash, an encoded dot segment, or a doubled
|
|
17
|
+
// slash all fail outright even though a normalizer could pick a reading for
|
|
18
|
+
// each. Picking a reading is how these bugs happen.
|
|
19
|
+
|
|
20
|
+
/** Forms on which a validator and a server could disagree. Checked on the raw
|
|
21
|
+
* string, before any decoding, because decoding is itself a reading. */
|
|
22
|
+
const FORBIDDEN_RAW = [
|
|
23
|
+
{ re: /%2f/i, why: 'an encoded path separator (%2f)' },
|
|
24
|
+
{ re: /%5c/i, why: 'an encoded backslash (%5c)' },
|
|
25
|
+
{ re: /\\/, why: 'a literal backslash' },
|
|
26
|
+
{ re: /%2e/i, why: 'an encoded dot segment (%2e)' },
|
|
27
|
+
{ re: /\/\//, why: 'a doubled slash' },
|
|
28
|
+
{ re: /%00/i, why: 'an encoded NUL' },
|
|
29
|
+
{ re: /[\u0000-\u001f\u007f]/, why: 'a control character' },
|
|
30
|
+
];
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Validate a `layer.parent.locator` as a relative reference, textually, before
|
|
34
|
+
* anything is fetched or resolved. 5.1.1: "A reader MUST reject a locator that
|
|
35
|
+
* violates this, at the manifest, before any fetch."
|
|
36
|
+
*/
|
|
37
|
+
export function validateLocatorShape(locator) {
|
|
38
|
+
if (typeof locator !== 'string' || locator.length === 0) {
|
|
39
|
+
throw new Error('locator must be a non-empty string');
|
|
40
|
+
}
|
|
41
|
+
if (locator.length > 1024) {
|
|
42
|
+
throw new Error('locator is implausibly long');
|
|
43
|
+
}
|
|
44
|
+
// Shape first, so the refusal names the actual violation: an absolute URL
|
|
45
|
+
// also trips the doubled-slash rule, and "contains a doubled slash" would
|
|
46
|
+
// be a confusing reason to reject "https://evil.example/x".
|
|
47
|
+
// MUST be a relative reference: no scheme, no authority.
|
|
48
|
+
if (/^[a-z][a-z0-9+.-]*:/i.test(locator)) {
|
|
49
|
+
throw new Error('locator must be a relative reference with no scheme');
|
|
50
|
+
}
|
|
51
|
+
if (locator.startsWith('//')) {
|
|
52
|
+
throw new Error('locator must have no authority');
|
|
53
|
+
}
|
|
54
|
+
for (const { re, why } of FORBIDDEN_RAW) {
|
|
55
|
+
if (re.test(locator)) throw new Error(`locator contains ${why}; refused rather than normalized`);
|
|
56
|
+
}
|
|
57
|
+
// An absolute path escapes the child's directory by construction.
|
|
58
|
+
if (locator.startsWith('/')) {
|
|
59
|
+
throw new Error('locator must be relative, not root-absolute');
|
|
60
|
+
}
|
|
61
|
+
// Dot segments in any form: a locator names a sibling, never a traversal.
|
|
62
|
+
const segments = locator.split('/');
|
|
63
|
+
for (const seg of segments) {
|
|
64
|
+
if (seg === '..' || seg === '.') {
|
|
65
|
+
throw new Error(`locator contains the dot segment ${JSON.stringify(seg)}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
if (locator.endsWith('/')) {
|
|
69
|
+
throw new Error('locator must name a file, not a directory');
|
|
70
|
+
}
|
|
71
|
+
if (locator.includes('?') || locator.includes('#')) {
|
|
72
|
+
throw new Error('locator must carry no query or fragment');
|
|
73
|
+
}
|
|
74
|
+
return true;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** The directory part of a URL path, with its trailing slash. */
|
|
78
|
+
function directoryOf(pathname) {
|
|
79
|
+
const at = pathname.lastIndexOf('/');
|
|
80
|
+
return at === -1 ? '/' : pathname.slice(0, at + 1);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Resolve a locator against the child's own URL and confirm the result stays
|
|
85
|
+
* within the child's origin and directory.
|
|
86
|
+
*
|
|
87
|
+
* @param {string} locator a locator that already passed validateLocatorShape
|
|
88
|
+
* @param {string} childUrl the URL the CHILD's bytes were fetched from
|
|
89
|
+
* @returns {string} the absolute URL to fetch the parent from
|
|
90
|
+
*/
|
|
91
|
+
export function resolveLocatorUrl(locator, childUrl) {
|
|
92
|
+
validateLocatorShape(locator);
|
|
93
|
+
let child;
|
|
94
|
+
try { child = new URL(childUrl); } catch (err) {
|
|
95
|
+
throw new Error(`child location ${JSON.stringify(childUrl)} is not a URL`, { cause: err });
|
|
96
|
+
}
|
|
97
|
+
if (child.protocol !== 'http:' && child.protocol !== 'https:') {
|
|
98
|
+
throw new Error(`parent resolution supports http and https only, not ${child.protocol}`);
|
|
99
|
+
}
|
|
100
|
+
const resolved = new URL(locator, child);
|
|
101
|
+
// Canonical comparison: URL already lowercases scheme and host and removes
|
|
102
|
+
// dot segments. The shape check above refused the forms where that
|
|
103
|
+
// normalization could disagree with a server's reading.
|
|
104
|
+
assertConfined(resolved, child, 'locator');
|
|
105
|
+
return resolved.href;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* The confinement rule, applied to a locator's result and to EVERY redirect
|
|
110
|
+
* hop. 5.1.1: "Redirects encountered while fetching a parent are subject to the
|
|
111
|
+
* same confinement as the locator, not a weaker one ... and the check is
|
|
112
|
+
* repeated at every hop before the redirected request is issued."
|
|
113
|
+
*
|
|
114
|
+
* A same-origin redirect to a path outside the directory fails exactly as a
|
|
115
|
+
* cross-origin one does.
|
|
116
|
+
*/
|
|
117
|
+
export function assertConfined(target, child, what = 'redirect target') {
|
|
118
|
+
const t = target instanceof URL ? target : new URL(target);
|
|
119
|
+
const c = child instanceof URL ? child : new URL(child);
|
|
120
|
+
if (t.protocol !== c.protocol) {
|
|
121
|
+
throw new Error(`${what} changes scheme from ${c.protocol} to ${t.protocol}`);
|
|
122
|
+
}
|
|
123
|
+
if (t.host !== c.host) {
|
|
124
|
+
throw new Error(`${what} leaves the child's origin: ${t.host} is not ${c.host}`);
|
|
125
|
+
}
|
|
126
|
+
if (t.username || t.password) {
|
|
127
|
+
throw new Error(`${what} carries credentials`);
|
|
128
|
+
}
|
|
129
|
+
for (const { re, why } of FORBIDDEN_RAW) {
|
|
130
|
+
// Re-checked on the resolved href: a redirect's Location header is a
|
|
131
|
+
// fresh string from the network, not the validated locator.
|
|
132
|
+
if (re.test(t.pathname)) throw new Error(`${what} contains ${why}`);
|
|
133
|
+
}
|
|
134
|
+
const dir = directoryOf(c.pathname);
|
|
135
|
+
if (!t.pathname.startsWith(dir)) {
|
|
136
|
+
throw new Error(`${what} path ${t.pathname} is not under the child's directory ${dir}`);
|
|
137
|
+
}
|
|
138
|
+
if (t.pathname.length === dir.length) {
|
|
139
|
+
throw new Error(`${what} names the directory itself, not a file`);
|
|
140
|
+
}
|
|
141
|
+
return true;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* The file-source analogue. 5.1.1 requires the check to run on the REAL path,
|
|
146
|
+
* "with every symbolic link followed", so a symlink cannot escape the
|
|
147
|
+
* directory. The caller supplies a realpath function because this module stays
|
|
148
|
+
* environment-neutral (no node:fs import in a file the browser bundle pulls in).
|
|
149
|
+
*
|
|
150
|
+
* @param {string} locator
|
|
151
|
+
* @param {string} childPath the child artifact's path
|
|
152
|
+
* @param {{realpath: (p: string) => Promise<string>, dirname: (p: string) => string,
|
|
153
|
+
* join: (...p: string[]) => string, sep?: string}} fsops
|
|
154
|
+
*/
|
|
155
|
+
export async function resolveLocatorPath(locator, childPath, fsops) {
|
|
156
|
+
validateLocatorShape(locator);
|
|
157
|
+
if (!fsops || typeof fsops.realpath !== 'function') {
|
|
158
|
+
throw new Error('resolveLocatorPath needs a realpath implementation');
|
|
159
|
+
}
|
|
160
|
+
const childDir = fsops.dirname(childPath);
|
|
161
|
+
// The child's own directory, resolved through links first: comparing against
|
|
162
|
+
// an unresolved directory would let a link in the PARENT position pass.
|
|
163
|
+
const realChildDir = await fsops.realpath(childDir);
|
|
164
|
+
const candidate = fsops.join(childDir, locator);
|
|
165
|
+
let realCandidate;
|
|
166
|
+
try {
|
|
167
|
+
realCandidate = await fsops.realpath(candidate);
|
|
168
|
+
} catch (err) {
|
|
169
|
+
// A locator naming something that does not exist is an availability
|
|
170
|
+
// failure (3.2), reported as such rather than as a security refusal.
|
|
171
|
+
throw new Error(`parent named by locator ${JSON.stringify(locator)} cannot be located`, { cause: err });
|
|
172
|
+
}
|
|
173
|
+
const sep = fsops.sep || '/';
|
|
174
|
+
const prefix = realChildDir.endsWith(sep) ? realChildDir : realChildDir + sep;
|
|
175
|
+
if (!realCandidate.startsWith(prefix)) {
|
|
176
|
+
throw new Error(`locator resolves to ${realCandidate}, outside the child's real directory ${realChildDir}`);
|
|
177
|
+
}
|
|
178
|
+
return realCandidate;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Resolution order of 5.1.1: lineage listing, then locator, then host resolver.
|
|
183
|
+
* Whatever located the bytes, the identity check of 3.2 is what admits them —
|
|
184
|
+
* so this returns a *candidate location only*, never a verdict on the bytes.
|
|
185
|
+
*
|
|
186
|
+
* A lineage listing is operator-supplied (the operator chose the shelf) and MAY
|
|
187
|
+
* name any origin the operator trusts; a manifest locator is publisher-supplied
|
|
188
|
+
* and gets the confinement above. That asymmetry is deliberate.
|
|
189
|
+
*
|
|
190
|
+
* @param {{identity: string, locator: string|null}} want
|
|
191
|
+
* @param {{lineage?: Map<string,string>|null, childLocation?: string|null,
|
|
192
|
+
* kind?: 'url'|'file', resolveParents?: boolean,
|
|
193
|
+
* hostResolver?: ((identity: string) => string|null)|null,
|
|
194
|
+
* fsops?: object|null}} opts
|
|
195
|
+
*/
|
|
196
|
+
export async function resolveParentLocation(want, opts = {}) {
|
|
197
|
+
const {
|
|
198
|
+
lineage = null, childLocation = null, kind = 'url',
|
|
199
|
+
resolveParents = true, hostResolver = null, fsops = null,
|
|
200
|
+
} = opts;
|
|
201
|
+
if (!want || typeof want.identity !== 'string') throw new Error('resolveParentLocation needs the wanted identity');
|
|
202
|
+
|
|
203
|
+
// 1. Lineage listing: operator-supplied, trusted for location.
|
|
204
|
+
if (lineage && lineage.has(want.identity)) {
|
|
205
|
+
return { location: lineage.get(want.identity), via: 'lineage' };
|
|
206
|
+
}
|
|
207
|
+
// 2. Manifest locator: publisher-supplied, confined.
|
|
208
|
+
if (want.locator) {
|
|
209
|
+
if (!resolveParents) {
|
|
210
|
+
throw new Error('locator resolution is disabled (resolveParents: false) and no lineage listing names this parent');
|
|
211
|
+
}
|
|
212
|
+
if (!childLocation) {
|
|
213
|
+
throw new Error('a locator needs the child\'s own location to resolve against');
|
|
214
|
+
}
|
|
215
|
+
const location = kind === 'file'
|
|
216
|
+
? await resolveLocatorPath(want.locator, childLocation, fsops)
|
|
217
|
+
: resolveLocatorUrl(want.locator, childLocation);
|
|
218
|
+
return { location, via: 'locator' };
|
|
219
|
+
}
|
|
220
|
+
// 3. Host resolver.
|
|
221
|
+
if (typeof hostResolver === 'function') {
|
|
222
|
+
const location = hostResolver(want.identity);
|
|
223
|
+
if (location) return { location, via: 'host' };
|
|
224
|
+
}
|
|
225
|
+
// 3.2: a reader that cannot locate an ancestor MUST fail the mount
|
|
226
|
+
// explicitly — never serve the layers it did reach as a partial corpus.
|
|
227
|
+
throw new Error(`cannot locate chain member ${want.identity.slice(0, 12)}…: no lineage entry, no locator and no host resolver`);
|
|
228
|
+
}
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
// Layer manifest validation — LAYERED_PROFILE.md 4.1, 4.2, 3.3.
|
|
2
|
+
//
|
|
3
|
+
// A layer is a complete-profile container with a `layer` object and profile
|
|
4
|
+
// string `pikelet-layer-v1`. This module validates that object's shape
|
|
5
|
+
// standalone, and then validates it against the parent a reader ACTUALLY
|
|
6
|
+
// OPENED (3.3): locators are hints, identities are the truth, and every
|
|
7
|
+
// arithmetic relation is re-derived rather than trusted.
|
|
8
|
+
//
|
|
9
|
+
// Nothing here reads bytes. The caller supplies a parsed manifest and, for the
|
|
10
|
+
// relational checks, the parent's parsed manifest plus the identity the parent
|
|
11
|
+
// was actually verified under. Keeping it pure makes the whole hostile-chain
|
|
12
|
+
// family of section 10 testable without building files.
|
|
13
|
+
|
|
14
|
+
export const LAYER_PROFILE = 'pikelet-layer-v1';
|
|
15
|
+
// 2.7: 1 <= depth <= 16. The producer default is lower (8, still unmeasured —
|
|
16
|
+
// see 13.5); this is the format ceiling a reader must refuse past.
|
|
17
|
+
export const MAX_DEPTH = 16;
|
|
18
|
+
// 3.3: a chain holds at most 2^31 - 1 records, ids 0 .. 2^31 - 2.
|
|
19
|
+
export const MAX_RECORDS = 2 ** 31 - 1;
|
|
20
|
+
// 3.3: warn once a chain's row total passes 2^30 so compaction can be
|
|
21
|
+
// scheduled before appends start failing.
|
|
22
|
+
export const ROW_TOTAL_WARN = 2 ** 30;
|
|
23
|
+
|
|
24
|
+
const isHex64 = (v) => typeof v === 'string' && /^[0-9a-f]{64}$/.test(v);
|
|
25
|
+
const isCount = (v, max = MAX_RECORDS) => Number.isSafeInteger(v) && v >= 0 && v <= max;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Canonical JSON, for the equality comparisons 4.1 requires on `encoder` and
|
|
29
|
+
* `layer.ingest`. Object keys sorted, no insignificant whitespace. Must match
|
|
30
|
+
* the builder's canonicalization or the equality checks are meaningless.
|
|
31
|
+
*/
|
|
32
|
+
export function canonicalJson(value) {
|
|
33
|
+
if (value === null || typeof value !== 'object') return JSON.stringify(value);
|
|
34
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`;
|
|
35
|
+
const keys = Object.keys(value).filter((k) => value[k] !== undefined).sort();
|
|
36
|
+
return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalJson(value[k])}`).join(',')}}`;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Validate a layer manifest's own `layer` object — everything checkable
|
|
41
|
+
* without the parent. Throws on the first violation; the message names the
|
|
42
|
+
* field so a hostile chain's refusal says which rule it broke.
|
|
43
|
+
*
|
|
44
|
+
* @returns {{identityFields: {parentIdentity: string, baseIdentity: string},
|
|
45
|
+
* depth: number, rowBase: number, records: number,
|
|
46
|
+
* tombstones: number, supersessions: number,
|
|
47
|
+
* tombstoneOnly: boolean, locator: string|null}}
|
|
48
|
+
*/
|
|
49
|
+
export function validateLayerObject(manifest) {
|
|
50
|
+
if (!manifest || typeof manifest !== 'object') throw new Error('layer manifest is not an object');
|
|
51
|
+
if (manifest.profile !== LAYER_PROFILE) {
|
|
52
|
+
throw new Error(`layer manifest profile is ${JSON.stringify(manifest.profile)}, expected ${LAYER_PROFILE}`);
|
|
53
|
+
}
|
|
54
|
+
const layer = manifest.layer;
|
|
55
|
+
if (!layer || typeof layer !== 'object' || Array.isArray(layer)) {
|
|
56
|
+
throw new Error('layer manifest has no layer object');
|
|
57
|
+
}
|
|
58
|
+
const parent = layer.parent;
|
|
59
|
+
if (!parent || typeof parent !== 'object' || Array.isArray(parent)) {
|
|
60
|
+
throw new Error('layer.parent is missing');
|
|
61
|
+
}
|
|
62
|
+
if (!isHex64(parent.identity)) {
|
|
63
|
+
throw new Error('layer.parent.identity must be 64 lowercase hex characters');
|
|
64
|
+
}
|
|
65
|
+
if (parent.locator !== undefined && typeof parent.locator !== 'string') {
|
|
66
|
+
throw new Error('layer.parent.locator, when present, must be a string');
|
|
67
|
+
}
|
|
68
|
+
if (!isHex64(layer.baseIdentity)) {
|
|
69
|
+
throw new Error('layer.baseIdentity must be 64 lowercase hex characters');
|
|
70
|
+
}
|
|
71
|
+
if (!Number.isSafeInteger(layer.depth) || layer.depth < 1 || layer.depth > MAX_DEPTH) {
|
|
72
|
+
throw new Error(`layer.depth must be an integer in [1, ${MAX_DEPTH}], got ${layer.depth}`);
|
|
73
|
+
}
|
|
74
|
+
if (!isCount(layer.rowBase)) {
|
|
75
|
+
throw new Error(`layer.rowBase must be an integer in [0, ${MAX_RECORDS}], got ${layer.rowBase}`);
|
|
76
|
+
}
|
|
77
|
+
if (!isCount(layer.records)) {
|
|
78
|
+
throw new Error(`layer.records must be an integer in [0, ${MAX_RECORDS}], got ${layer.records}`);
|
|
79
|
+
}
|
|
80
|
+
// 3.3: every rowBase + records sum is bounded by the id ceiling.
|
|
81
|
+
if (layer.rowBase + layer.records > MAX_RECORDS) {
|
|
82
|
+
throw new Error(`layer.rowBase + layer.records (${layer.rowBase + layer.records}) exceeds ${MAX_RECORDS}`);
|
|
83
|
+
}
|
|
84
|
+
// 3.4: the cumulative bitset covers [0, rowBase), so it cannot hold more
|
|
85
|
+
// bits than that, and a layer cannot tombstone its own rows.
|
|
86
|
+
if (!isCount(layer.tombstones) || layer.tombstones > layer.rowBase) {
|
|
87
|
+
throw new Error(`layer.tombstones must be an integer in [0, rowBase=${layer.rowBase}], got ${layer.tombstones}`);
|
|
88
|
+
}
|
|
89
|
+
if (!isCount(layer.supersessions, layer.rowBase)) {
|
|
90
|
+
throw new Error(`layer.supersessions must be an integer in [0, rowBase=${layer.rowBase}], got ${layer.supersessions}`);
|
|
91
|
+
}
|
|
92
|
+
// 3.5: each supersession's oldId must be tombstoned in this layer's
|
|
93
|
+
// bitset, and oldId is unique within a layer, so there cannot be more
|
|
94
|
+
// edges than tombstones.
|
|
95
|
+
if (layer.supersessions > layer.tombstones) {
|
|
96
|
+
throw new Error(`layer.supersessions (${layer.supersessions}) exceeds layer.tombstones (${layer.tombstones}); every oldId must be tombstoned and is unique per layer`);
|
|
97
|
+
}
|
|
98
|
+
// 3.5: newId lies in this layer's own id range, so a layer with no
|
|
99
|
+
// records of its own can record no supersession.
|
|
100
|
+
if (layer.records === 0 && layer.supersessions > 0) {
|
|
101
|
+
throw new Error('a tombstone-only layer cannot record a supersession: newId must lie in its own id range');
|
|
102
|
+
}
|
|
103
|
+
if (layer.ingest === undefined || layer.ingest === null || typeof layer.ingest !== 'object' || Array.isArray(layer.ingest)) {
|
|
104
|
+
throw new Error('layer.ingest is missing; appendability requires an ingestion declaration (6.1)');
|
|
105
|
+
}
|
|
106
|
+
// 4.1: ingestAsserted is a SIBLING of layer.ingest, never a member of it.
|
|
107
|
+
// If it were a member, asserting a declaration would change its canonical
|
|
108
|
+
// form and a compacted base's corpus.ingest would not be byte-identical to
|
|
109
|
+
// the declaration the chain carried (6.3).
|
|
110
|
+
if ('ingestAsserted' in layer.ingest) {
|
|
111
|
+
throw new Error('layer.ingest must not contain ingestAsserted; it is a sibling field inside layer, not part of the declaration');
|
|
112
|
+
}
|
|
113
|
+
if (layer.ingestAsserted !== undefined && typeof layer.ingestAsserted !== 'boolean') {
|
|
114
|
+
throw new Error('layer.ingestAsserted, when present, must be a boolean');
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// 4.1: corpus.records MUST equal layer.records. A tombstone-only layer
|
|
118
|
+
// still carries a corpus object, with records 0 and provenance, and MUST
|
|
119
|
+
// omit the layout fields and the index object.
|
|
120
|
+
const corpus = manifest.corpus;
|
|
121
|
+
if (!corpus || typeof corpus !== 'object' || Array.isArray(corpus)) {
|
|
122
|
+
throw new Error('layer manifest has no corpus block');
|
|
123
|
+
}
|
|
124
|
+
if (corpus.records !== layer.records) {
|
|
125
|
+
throw new Error(`manifest corpus.records (${corpus.records}) must equal layer.records (${layer.records})`);
|
|
126
|
+
}
|
|
127
|
+
const tombstoneOnly = layer.records === 0;
|
|
128
|
+
const layoutFields = ['layout', 'pageRecords', 'pages', 'recordDigest', 'pageTableSha256'];
|
|
129
|
+
if (tombstoneOnly) {
|
|
130
|
+
for (const f of layoutFields) {
|
|
131
|
+
if (corpus[f] !== undefined) {
|
|
132
|
+
throw new Error(`a tombstone-only layer must omit corpus.${f}: it has no corpus segment to describe`);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
if (manifest.index !== undefined) {
|
|
136
|
+
throw new Error('a tombstone-only layer must omit the index object: it has no index segment');
|
|
137
|
+
}
|
|
138
|
+
} else {
|
|
139
|
+
if (typeof corpus.layout !== 'string') {
|
|
140
|
+
throw new Error('a layer with records must declare corpus.layout');
|
|
141
|
+
}
|
|
142
|
+
if (!manifest.index || typeof manifest.index !== 'object') {
|
|
143
|
+
throw new Error('a layer with records must carry an index object');
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
return {
|
|
148
|
+
identityFields: { parentIdentity: parent.identity, baseIdentity: layer.baseIdentity },
|
|
149
|
+
depth: layer.depth,
|
|
150
|
+
rowBase: layer.rowBase,
|
|
151
|
+
records: layer.records,
|
|
152
|
+
tombstones: layer.tombstones,
|
|
153
|
+
supersessions: layer.supersessions,
|
|
154
|
+
tombstoneOnly,
|
|
155
|
+
locator: typeof parent.locator === 'string' ? parent.locator : null,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* The relational checks of 3.3 and 4.1, against the parent a reader actually
|
|
161
|
+
* opened and verified. `parentIdentity` is the identity the parent's own bytes
|
|
162
|
+
* hashed to — never a value read out of the child.
|
|
163
|
+
*
|
|
164
|
+
* @param {object} childManifest
|
|
165
|
+
* @param {{manifest: object, identity: string, isBase: boolean}} parent
|
|
166
|
+
*/
|
|
167
|
+
export function validateAgainstParent(childManifest, parent) {
|
|
168
|
+
const child = validateLayerObject(childManifest);
|
|
169
|
+
if (!parent || typeof parent !== 'object') throw new Error('no parent supplied');
|
|
170
|
+
const { manifest: pm, identity: parentIdentity, isBase } = parent;
|
|
171
|
+
if (!isHex64(parentIdentity)) throw new Error('parent identity must be 64 lowercase hex characters');
|
|
172
|
+
|
|
173
|
+
// 3.2: refuse a chain in which any opened parent's identity differs from
|
|
174
|
+
// the identity its child committed to. This is the hash-chain link.
|
|
175
|
+
if (child.identityFields.parentIdentity !== parentIdentity) {
|
|
176
|
+
throw new Error(`layer commits to parent identity ${child.identityFields.parentIdentity.slice(0, 12)}… but the opened parent is ${parentIdentity.slice(0, 12)}…`);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const parentDepth = isBase ? 0 : pm?.layer?.depth;
|
|
180
|
+
const parentRowBase = isBase ? 0 : pm?.layer?.rowBase;
|
|
181
|
+
const parentRecords = pm?.corpus?.records;
|
|
182
|
+
if (!isBase) {
|
|
183
|
+
if (!Number.isSafeInteger(parentDepth)) throw new Error('parent layer.depth is not an integer');
|
|
184
|
+
if (!Number.isSafeInteger(parentRowBase)) throw new Error('parent layer.rowBase is not an integer');
|
|
185
|
+
}
|
|
186
|
+
if (!Number.isSafeInteger(parentRecords)) throw new Error('parent corpus.records is not an integer');
|
|
187
|
+
|
|
188
|
+
// 3.3, verified against the parent actually opened:
|
|
189
|
+
// layer.rowBase == parent.layer.rowBase + parent.corpus.records (parent a layer)
|
|
190
|
+
// layer.rowBase == parent.corpus.records (parent the base)
|
|
191
|
+
const expectedRowBase = isBase ? parentRecords : parentRowBase + parentRecords;
|
|
192
|
+
if (child.rowBase !== expectedRowBase) {
|
|
193
|
+
throw new Error(`layer.rowBase is ${child.rowBase} but the opened parent implies ${expectedRowBase}`);
|
|
194
|
+
}
|
|
195
|
+
if (child.depth !== parentDepth + 1) {
|
|
196
|
+
throw new Error(`layer.depth is ${child.depth} but the opened parent is at depth ${parentDepth}`);
|
|
197
|
+
}
|
|
198
|
+
// layer.baseIdentity == parent.layer.baseIdentity, or the parent's own
|
|
199
|
+
// identity when the parent is the base.
|
|
200
|
+
const expectedBase = isBase ? parentIdentity : pm?.layer?.baseIdentity;
|
|
201
|
+
if (child.identityFields.baseIdentity !== expectedBase) {
|
|
202
|
+
throw new Error(`layer.baseIdentity is ${String(child.identityFields.baseIdentity).slice(0, 12)}… but the chain's base is ${String(expectedBase).slice(0, 12)}…`);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// 4.1: encoder, dim and metric MUST equal the base's. Checked against the
|
|
206
|
+
// parent here, which gives the same result transitively along the chain and
|
|
207
|
+
// catches the difference at the layer that introduced it.
|
|
208
|
+
if (canonicalJson(childManifest.encoder) !== canonicalJson(pm?.encoder)) {
|
|
209
|
+
throw new Error('layer encoder object differs from its parent\'s; a chain may not change the encoder');
|
|
210
|
+
}
|
|
211
|
+
if (childManifest.dim !== pm?.dim) {
|
|
212
|
+
throw new Error(`layer dim ${childManifest.dim} differs from its parent's ${pm?.dim}`);
|
|
213
|
+
}
|
|
214
|
+
if (childManifest.metric !== pm?.metric) {
|
|
215
|
+
throw new Error(`layer metric ${JSON.stringify(childManifest.metric)} differs from its parent's ${JSON.stringify(pm?.metric)}`);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
return child;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* The chain ingestion declaration of 4.1/6.1: the base's `corpus.ingest` when
|
|
223
|
+
* the base carries one; otherwise the depth-1 layer's `layer.ingest`, which
|
|
224
|
+
* must then be marked `ingestAsserted`. Returns the canonical form to compare
|
|
225
|
+
* every layer against, plus whether the chain is asserted.
|
|
226
|
+
*/
|
|
227
|
+
export function chainIngestDeclaration(baseManifest, depth1Manifest) {
|
|
228
|
+
const baseIngest = baseManifest?.corpus?.ingest;
|
|
229
|
+
if (baseIngest !== undefined && baseIngest !== null) {
|
|
230
|
+
return { canonical: canonicalJson(baseIngest), asserted: false, source: 'base' };
|
|
231
|
+
}
|
|
232
|
+
if (!depth1Manifest) {
|
|
233
|
+
throw new Error('the base carries no corpus.ingest and no depth-1 layer was supplied: the chain has no ingestion declaration (6.1)');
|
|
234
|
+
}
|
|
235
|
+
const layer = depth1Manifest.layer || {};
|
|
236
|
+
if (layer.ingestAsserted !== true) {
|
|
237
|
+
throw new Error('the base carries no corpus.ingest, so the depth-1 layer must set layer.ingestAsserted true to introduce one (6.1)');
|
|
238
|
+
}
|
|
239
|
+
return { canonical: canonicalJson(layer.ingest), asserted: true, source: 'depth-1' };
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Check one layer's declaration against the chain's. Only the depth-1 layer of
|
|
244
|
+
* an assertion-bearing chain can introduce the assertion; no later layer may
|
|
245
|
+
* introduce, drop or change one.
|
|
246
|
+
*/
|
|
247
|
+
export function validateIngestAgainstChain(layerManifest, chainDecl) {
|
|
248
|
+
const layer = layerManifest?.layer || {};
|
|
249
|
+
if (canonicalJson(layer.ingest) !== chainDecl.canonical) {
|
|
250
|
+
throw new Error(`layer at depth ${layer.depth} carries an ingestion declaration that differs from the chain's (6.1)`);
|
|
251
|
+
}
|
|
252
|
+
const asserted = layer.ingestAsserted === true;
|
|
253
|
+
if (chainDecl.asserted && !asserted) {
|
|
254
|
+
throw new Error(`layer at depth ${layer.depth} must carry ingestAsserted true on an asserted chain`);
|
|
255
|
+
}
|
|
256
|
+
if (!chainDecl.asserted && asserted) {
|
|
257
|
+
throw new Error(`layer at depth ${layer.depth} asserts an ingestion declaration, but the chain's base declares one`);
|
|
258
|
+
}
|
|
259
|
+
return true;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* `rowTotal(A) = A.rowBase + A.records` (3.1) — the number of ids that existed
|
|
264
|
+
* when A was the head. For a base it is just its record count.
|
|
265
|
+
*/
|
|
266
|
+
export function rowTotal(manifest) {
|
|
267
|
+
const rb = manifest?.layer?.rowBase;
|
|
268
|
+
const records = manifest?.corpus?.records;
|
|
269
|
+
if (!Number.isSafeInteger(records)) throw new Error('manifest corpus.records is not an integer');
|
|
270
|
+
return (Number.isSafeInteger(rb) ? rb : 0) + records;
|
|
271
|
+
}
|