zarr-metadata 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/semantics.d.ts +9 -3
- package/dist/semantics.d.ts.map +1 -1
- package/dist/semantics.js +94 -19
- package/dist/semantics.js.map +1 -1
- package/package.json +1 -1
- package/src/semantics.ts +92 -19
package/dist/semantics.d.ts
CHANGED
|
@@ -8,12 +8,18 @@
|
|
|
8
8
|
*
|
|
9
9
|
* - the `regular` chunk grid's `chunk_shape` has one length per dimension
|
|
10
10
|
* of `shape`;
|
|
11
|
+
* - the `rectilinear` chunk grid's `chunk_shapes` has one entry per
|
|
12
|
+
* dimension, and each explicit chunk list (integers and `[size, count]`
|
|
13
|
+
* run-length pairs) sums exactly to that dimension's length (the
|
|
14
|
+
* bare-integer uniform shorthand carries no sum constraint, like the
|
|
15
|
+
* regular grid);
|
|
11
16
|
* - a `transpose` codec's `order` is a permutation of `0..n-1`, with one
|
|
12
17
|
* entry per array dimension;
|
|
13
18
|
* - a `sharding_indexed` codec's inner `chunk_shape` matches the array's
|
|
14
|
-
* dimensionality and evenly divides
|
|
15
|
-
*
|
|
16
|
-
* chunk when sharding
|
|
19
|
+
* dimensionality and evenly divides every chunk it shards — the grid's
|
|
20
|
+
* chunk at the top level (each distinct per-dimension size, for
|
|
21
|
+
* rectilinear grids), the parent shard's inner chunk when sharding
|
|
22
|
+
* nests;
|
|
17
23
|
* - `fill_value` has a JSON shape permitted for the named core data type
|
|
18
24
|
* (booleans for `bool`, ranged integers for the int types, numbers /
|
|
19
25
|
* the "NaN"-family sentinels / width-checked "0x…" strings for the
|
package/dist/semantics.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"semantics.d.ts","sourceRoot":"","sources":["../src/semantics.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"semantics.d.ts","sourceRoot":"","sources":["../src/semantics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,EAAU,KAAK,SAAS,EAAoB,MAAM,aAAa,CAAC;AAuMvE;;;;;GAKG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,OAAO,GAAG,SAAS,CAqFlE"}
|
package/dist/semantics.js
CHANGED
|
@@ -8,12 +8,18 @@
|
|
|
8
8
|
*
|
|
9
9
|
* - the `regular` chunk grid's `chunk_shape` has one length per dimension
|
|
10
10
|
* of `shape`;
|
|
11
|
+
* - the `rectilinear` chunk grid's `chunk_shapes` has one entry per
|
|
12
|
+
* dimension, and each explicit chunk list (integers and `[size, count]`
|
|
13
|
+
* run-length pairs) sums exactly to that dimension's length (the
|
|
14
|
+
* bare-integer uniform shorthand carries no sum constraint, like the
|
|
15
|
+
* regular grid);
|
|
11
16
|
* - a `transpose` codec's `order` is a permutation of `0..n-1`, with one
|
|
12
17
|
* entry per array dimension;
|
|
13
18
|
* - a `sharding_indexed` codec's inner `chunk_shape` matches the array's
|
|
14
|
-
* dimensionality and evenly divides
|
|
15
|
-
*
|
|
16
|
-
* chunk when sharding
|
|
19
|
+
* dimensionality and evenly divides every chunk it shards — the grid's
|
|
20
|
+
* chunk at the top level (each distinct per-dimension size, for
|
|
21
|
+
* rectilinear grids), the parent shard's inner chunk when sharding
|
|
22
|
+
* nests;
|
|
17
23
|
* - `fill_value` has a JSON shape permitted for the named core data type
|
|
18
24
|
* (booleans for `bool`, ranged integers for the int types, numbers /
|
|
19
25
|
* the "NaN"-family sentinels / width-checked "0x…" strings for the
|
|
@@ -122,10 +128,12 @@ function isPermutation(order) {
|
|
|
122
128
|
}
|
|
123
129
|
/**
|
|
124
130
|
* Walk one codec pipeline. `dims` is the array dimensionality at this point
|
|
125
|
-
* (undefined when unknowable); `
|
|
126
|
-
* this pipeline
|
|
131
|
+
* (undefined when unknowable); `outerChunkSizes` holds the distinct chunk
|
|
132
|
+
* lengths per dimension this pipeline may encode — what a sharding codec's
|
|
133
|
+
* inner chunks must divide. A regular grid contributes one size per
|
|
134
|
+
* dimension; a rectilinear grid may contribute several.
|
|
127
135
|
*/
|
|
128
|
-
function pipelineIssues(pipeline, path, dims,
|
|
136
|
+
function pipelineIssues(pipeline, path, dims, outerChunkSizes) {
|
|
129
137
|
const issues = [];
|
|
130
138
|
pipeline.forEach((entry, index) => {
|
|
131
139
|
const parts = fieldParts(entry);
|
|
@@ -164,19 +172,32 @@ function pipelineIssues(pipeline, path, dims, outerChunkShape) {
|
|
|
164
172
|
kind: "invalid_value",
|
|
165
173
|
});
|
|
166
174
|
}
|
|
167
|
-
else if (
|
|
168
|
-
chunkShape.length ===
|
|
169
|
-
chunkShape.every((length) => length > 0)
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
+
else if (outerChunkSizes !== undefined &&
|
|
176
|
+
chunkShape.length === outerChunkSizes.length &&
|
|
177
|
+
chunkShape.every((length) => length > 0)) {
|
|
178
|
+
let violation;
|
|
179
|
+
outerChunkSizes.forEach((sizes, axis) => {
|
|
180
|
+
if (violation !== undefined)
|
|
181
|
+
return;
|
|
182
|
+
const bad = sizes.find((size) => size % chunkShape[axis] !== 0);
|
|
183
|
+
if (bad !== undefined)
|
|
184
|
+
violation = { axis, size: bad };
|
|
175
185
|
});
|
|
186
|
+
if (violation !== undefined) {
|
|
187
|
+
const { axis, size } = violation;
|
|
188
|
+
const uniform = outerChunkSizes.every((sizes) => sizes.length === 1);
|
|
189
|
+
issues.push({
|
|
190
|
+
path: chunkShapePath,
|
|
191
|
+
message: uniform
|
|
192
|
+
? `expected ${JSON.stringify(chunkShape)} to evenly divide the outer chunk shape ${JSON.stringify(outerChunkSizes.map((sizes) => sizes[0]))}`
|
|
193
|
+
: `expected ${JSON.stringify(chunkShape)} to evenly divide every chunk size of the grid (dimension ${axis} has chunk size ${size})`,
|
|
194
|
+
kind: "invalid_value",
|
|
195
|
+
});
|
|
196
|
+
}
|
|
176
197
|
}
|
|
177
198
|
const inner = configuration?.["codecs"];
|
|
178
199
|
if (Array.isArray(inner)) {
|
|
179
|
-
issues.push(...pipelineIssues(inner, [...path, index, "configuration", "codecs"], dims, chunkShape));
|
|
200
|
+
issues.push(...pipelineIssues(inner, [...path, index, "configuration", "codecs"], dims, chunkShape.map((length) => [length])));
|
|
180
201
|
}
|
|
181
202
|
// index_codecs encode the shard index, whose shape differs from the
|
|
182
203
|
// array's — no dimensional context applies there.
|
|
@@ -197,18 +218,72 @@ export function validateArraySemanticsV3(value) {
|
|
|
197
218
|
const shape = isIntArray(value["shape"]) ? value["shape"] : undefined;
|
|
198
219
|
const dims = shape?.length;
|
|
199
220
|
const grid = fieldParts(value["chunk_grid"]);
|
|
200
|
-
let
|
|
221
|
+
let chunkSizes;
|
|
201
222
|
if (grid?.name === "regular") {
|
|
202
223
|
const configured = grid.configuration?.["chunk_shape"];
|
|
203
224
|
if (isIntArray(configured)) {
|
|
204
|
-
chunkShape = configured;
|
|
205
225
|
if (dims !== undefined && configured.length !== dims) {
|
|
206
226
|
issues.push({
|
|
207
227
|
path: ["chunk_grid", "configuration", "chunk_shape"],
|
|
208
228
|
message: `expected one length per dimension of shape (${dims})`,
|
|
209
229
|
kind: "invalid_value",
|
|
210
230
|
});
|
|
211
|
-
|
|
231
|
+
// wrong arity: unusable as division context
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
chunkSizes = configured.map((length) => [length]);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
else if (grid?.name === "rectilinear") {
|
|
239
|
+
const specs = grid.configuration?.["chunk_shapes"];
|
|
240
|
+
if (Array.isArray(specs)) {
|
|
241
|
+
if (dims !== undefined && specs.length !== dims) {
|
|
242
|
+
issues.push({
|
|
243
|
+
path: ["chunk_grid", "configuration", "chunk_shapes"],
|
|
244
|
+
message: `expected one entry per dimension of shape (${dims})`,
|
|
245
|
+
kind: "invalid_value",
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
else {
|
|
249
|
+
const perDim = specs.map((spec, dim) => {
|
|
250
|
+
// Bare integer: uniform shorthand, no sum constraint (edge chunks
|
|
251
|
+
// are permitted, as with the regular grid).
|
|
252
|
+
if (Number.isInteger(spec))
|
|
253
|
+
return spec > 0 ? [spec] : undefined;
|
|
254
|
+
if (!Array.isArray(spec))
|
|
255
|
+
return undefined; // registry schema's problem
|
|
256
|
+
const sizes = new Set();
|
|
257
|
+
let total = 0;
|
|
258
|
+
for (const entry of spec) {
|
|
259
|
+
if (Number.isInteger(entry)) {
|
|
260
|
+
sizes.add(entry);
|
|
261
|
+
total += entry;
|
|
262
|
+
}
|
|
263
|
+
else if (Array.isArray(entry) &&
|
|
264
|
+
entry.length === 2 &&
|
|
265
|
+
Number.isInteger(entry[0]) &&
|
|
266
|
+
Number.isInteger(entry[1])) {
|
|
267
|
+
sizes.add(entry[0]);
|
|
268
|
+
total += entry[0] * entry[1];
|
|
269
|
+
}
|
|
270
|
+
else {
|
|
271
|
+
return undefined; // malformed entry: registry schema's problem
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
const extent = shape?.[dim];
|
|
275
|
+
if (extent !== undefined && total !== extent) {
|
|
276
|
+
issues.push({
|
|
277
|
+
path: ["chunk_grid", "configuration", "chunk_shapes", dim],
|
|
278
|
+
message: `expected chunk sizes summing to ${extent} along dimension ${dim}, got ${total}`,
|
|
279
|
+
kind: "invalid_value",
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
return [...sizes];
|
|
283
|
+
});
|
|
284
|
+
if (perDim.every((sizes) => sizes !== undefined)) {
|
|
285
|
+
chunkSizes = perDim;
|
|
286
|
+
}
|
|
212
287
|
}
|
|
213
288
|
}
|
|
214
289
|
}
|
|
@@ -221,7 +296,7 @@ export function validateArraySemanticsV3(value) {
|
|
|
221
296
|
}
|
|
222
297
|
const codecs = value["codecs"];
|
|
223
298
|
if (Array.isArray(codecs)) {
|
|
224
|
-
issues.push(...pipelineIssues(codecs, ["codecs"], dims,
|
|
299
|
+
issues.push(...pipelineIssues(codecs, ["codecs"], dims, chunkSizes));
|
|
225
300
|
}
|
|
226
301
|
return treeOf(issues);
|
|
227
302
|
}
|
package/dist/semantics.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"semantics.js","sourceRoot":"","sources":["../src/semantics.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"semantics.js","sourceRoot":"","sources":["../src/semantics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,EAAE,MAAM,EAAoC,MAAM,aAAa,CAAC;AAIvE,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtF,MAAM,KAAK,GAAY,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;IACpD,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,MAAM,CAAC,SAAS,CAAC;AACtD,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,CACL,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACpB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;QAC1C,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAC9C,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,SAAS,UAAU,CACjB,KAAc;IAEd,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC;IAChF,IAAI,aAAa,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,QAAQ,EAAE,CAAC;QAC9D,MAAM,aAAa,GAAG,KAAK,CAAC,eAAe,CAAC,CAAC;QAC7C,OAAO;YACL,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;YACnB,aAAa,EAAE,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;SACxE,CAAC;IACJ,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,6EAA6E;AAE7E,MAAM,UAAU,GAA8C;IAC5D,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC;IACjB,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC;IACtB,KAAK,EAAE,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC;IAChC,KAAK,EAAE,CAAC,CAAC,mBAAmB,EAAE,mBAAmB,CAAC;IAClD,KAAK,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC;IACf,MAAM,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC;IAClB,MAAM,EAAE,CAAC,CAAC,EAAE,UAAU,CAAC;IACvB,MAAM,EAAE,CAAC,CAAC,EAAE,oBAAoB,CAAC;CAClC,CAAC;AAEF,MAAM,gBAAgB,GAA2B;IAC/C,OAAO,EAAE,CAAC;IACV,OAAO,EAAE,CAAC;IACV,OAAO,EAAE,EAAE;CACZ,CAAC;AAEF,+DAA+D;AAC/D,MAAM,4BAA4B,GAA2B;IAC3D,SAAS,EAAE,CAAC;IACZ,UAAU,EAAE,EAAE;CACf,CAAC;AAEF,SAAS,WAAW,CAAC,KAAc,EAAE,SAAiB;IACpD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,UAAU,IAAI,KAAK,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAClF,OAAO,IAAI,MAAM,CAAC,kBAAkB,SAAS,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACjE,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY,EAAE,SAAiB;IACvD,OAAO,CACL,0DAA0D;QAC1D,GAAG,SAAS,2CAA2C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAC9E,CAAC;AACJ,CAAC;AAED,oFAAoF;AACpF,SAAS,cAAc,CAAC,YAAoB,EAAE,IAAa;IACzD,IAAI,YAAY,KAAK,MAAM,EAAE,CAAC;QAC5B,OAAO,OAAO,IAAI,KAAK,SAAS;YAC9B,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,oDAAoD,CAAC;IAC3D,CAAC;IACD,MAAM,QAAQ,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC;IAC1C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,QAAQ,CAAC;QAC7B,OAAO,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAK,IAAe,IAAI,GAAG,IAAK,IAAe,IAAI,IAAI;YAClF,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,2BAA2B,GAAG,KAAK,IAAI,mBAAmB,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE,CAAC;IAC/F,CAAC;IACD,MAAM,WAAW,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC;IACnD,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC;YACnC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,gBAAgB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAClD,CAAC;IACD,MAAM,aAAa,GAAG,4BAA4B,CAAC,YAAY,CAAC,CAAC;IACjE,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,EAAE,GACN,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;YACnB,IAAI,CAAC,MAAM,KAAK,CAAC;YACjB,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC;QACzD,OAAO,EAAE;YACP,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,gEAAgE,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE,CAAC;IACrG,CAAC;IACD,OAAO,SAAS,CAAC,CAAC,qCAAqC;AACzD,CAAC;AAED,8EAA8E;AAE9E,SAAS,aAAa,CAAC,KAAe;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC5B,OAAO,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;AAClG,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CACrB,QAAmB,EACnB,IAAU,EACV,IAAwB,EACxB,eAAuC;IAEvC,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAChC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO;QAChC,MAAM,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,KAAK,CAAC;QACtC,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,aAAa,EAAE,CAAC,OAAO,CAAC,CAAC;YACvC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,CAAC,sCAAsC;YACtE,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;YAC7D,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,6CAA6C,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;oBACxE,IAAI,EAAE,eAAe;iBACtB,CAAC,CAAC;YACL,CAAC;YACD,IAAI,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;gBAChD,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,SAAS;oBACf,OAAO,EAAE,2CAA2C,IAAI,GAAG;oBAC3D,IAAI,EAAE,eAAe;iBACtB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,KAAK,kBAAkB,EAAE,CAAC;YACvC,MAAM,UAAU,GAAG,aAAa,EAAE,CAAC,aAAa,CAAC,CAAC;YAClD,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;gBAAE,OAAO;YACpC,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,aAAa,CAAC,CAAC;YACxE,IAAI,IAAI,KAAK,SAAS,IAAI,UAAU,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;gBACrD,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,cAAc;oBACpB,OAAO,EAAE,4CAA4C,IAAI,GAAG;oBAC5D,IAAI,EAAE,eAAe;iBACtB,CAAC,CAAC;YACL,CAAC;iBAAM,IACL,eAAe,KAAK,SAAS;gBAC7B,UAAU,CAAC,MAAM,KAAK,eAAe,CAAC,MAAM;gBAC5C,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,EACxC,CAAC;gBACD,IAAI,SAAqD,CAAC;gBAC1D,eAAe,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBACtC,IAAI,SAAS,KAAK,SAAS;wBAAE,OAAO;oBACpC,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,GAAI,UAAU,CAAC,IAAI,CAAY,KAAK,CAAC,CAAC,CAAC;oBAC5E,IAAI,GAAG,KAAK,SAAS;wBAAE,SAAS,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;gBACzD,CAAC,CAAC,CAAC;gBACH,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;oBAC5B,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC;oBACjC,MAAM,OAAO,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;oBACrE,MAAM,CAAC,IAAI,CAAC;wBACV,IAAI,EAAE,cAAc;wBACpB,OAAO,EAAE,OAAO;4BACd,CAAC,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,2CAA2C,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;4BAC7I,CAAC,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,6DAA6D,IAAI,mBAAmB,IAAI,GAAG;wBACrI,IAAI,EAAE,eAAe;qBACtB,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YACD,MAAM,KAAK,GAAG,aAAa,EAAE,CAAC,QAAQ,CAAC,CAAC;YACxC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CACT,GAAG,cAAc,CACf,KAAK,EACL,CAAC,GAAG,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,QAAQ,CAAC,EAC3C,IAAI,EACJ,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CACrC,CACF,CAAC;YACJ,CAAC;YACD,oEAAoE;YACpE,kDAAkD;QACpD,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,wBAAwB,CAAC,KAAc;IACrD,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,KAAK,OAAO;QAAE,OAAO,MAAM,CAAC,EAAE,CAAC,CAAC;IAC/E,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACtE,MAAM,IAAI,GAAG,KAAK,EAAE,MAAM,CAAC;IAE3B,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;IAC7C,IAAI,UAAkC,CAAC;IACvC,IAAI,IAAI,EAAE,IAAI,KAAK,SAAS,EAAE,CAAC;QAC7B,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,aAAa,CAAC,CAAC;QACvD,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC3B,IAAI,IAAI,KAAK,SAAS,IAAI,UAAU,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;gBACrD,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,CAAC,YAAY,EAAE,eAAe,EAAE,aAAa,CAAC;oBACpD,OAAO,EAAE,+CAA+C,IAAI,GAAG;oBAC/D,IAAI,EAAE,eAAe;iBACtB,CAAC,CAAC;gBACH,4CAA4C;YAC9C,CAAC;iBAAM,CAAC;gBACN,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;IACH,CAAC;SAAM,IAAI,IAAI,EAAE,IAAI,KAAK,aAAa,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC,cAAc,CAAC,CAAC;QACnD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,IAAI,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;gBAChD,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,CAAC,YAAY,EAAE,eAAe,EAAE,cAAc,CAAC;oBACrD,OAAO,EAAE,8CAA8C,IAAI,GAAG;oBAC9D,IAAI,EAAE,eAAe;iBACtB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAA6B,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE;oBAC/D,kEAAkE;oBAClE,4CAA4C;oBAC5C,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC;wBAAE,OAAO,IAAc,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAc,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;oBACrF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;wBAAE,OAAO,SAAS,CAAC,CAAC,4BAA4B;oBACxE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;oBAChC,IAAI,KAAK,GAAG,CAAC,CAAC;oBACd,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;wBACzB,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;4BAC5B,KAAK,CAAC,GAAG,CAAC,KAAe,CAAC,CAAC;4BAC3B,KAAK,IAAI,KAAe,CAAC;wBAC3B,CAAC;6BAAM,IACL,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;4BACpB,KAAK,CAAC,MAAM,KAAK,CAAC;4BAClB,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;4BAC1B,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAC1B,CAAC;4BACD,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAW,CAAC,CAAC;4BAC9B,KAAK,IAAK,KAAK,CAAC,CAAC,CAAY,GAAI,KAAK,CAAC,CAAC,CAAY,CAAC;wBACvD,CAAC;6BAAM,CAAC;4BACN,OAAO,SAAS,CAAC,CAAC,6CAA6C;wBACjE,CAAC;oBACH,CAAC;oBACD,MAAM,MAAM,GAAG,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC;oBAC5B,IAAI,MAAM,KAAK,SAAS,IAAI,KAAK,KAAK,MAAM,EAAE,CAAC;wBAC7C,MAAM,CAAC,IAAI,CAAC;4BACV,IAAI,EAAE,CAAC,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,GAAG,CAAC;4BAC1D,OAAO,EAAE,mCAAmC,MAAM,oBAAoB,GAAG,SAAS,KAAK,EAAE;4BACzF,IAAI,EAAE,eAAe;yBACtB,CAAC,CAAC;oBACL,CAAC;oBACD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;gBACpB,CAAC,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,EAAE,CAAC;oBACjD,UAAU,GAAG,MAAoB,CAAC;gBACpC,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;IAChD,IAAI,QAAQ,KAAK,SAAS,IAAI,YAAY,IAAI,KAAK,EAAE,CAAC;QACpD,MAAM,OAAO,GAAG,cAAc,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC;QACnE,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,YAAY,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/B,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC;AACxB,CAAC"}
|
package/package.json
CHANGED
package/src/semantics.ts
CHANGED
|
@@ -8,12 +8,18 @@
|
|
|
8
8
|
*
|
|
9
9
|
* - the `regular` chunk grid's `chunk_shape` has one length per dimension
|
|
10
10
|
* of `shape`;
|
|
11
|
+
* - the `rectilinear` chunk grid's `chunk_shapes` has one entry per
|
|
12
|
+
* dimension, and each explicit chunk list (integers and `[size, count]`
|
|
13
|
+
* run-length pairs) sums exactly to that dimension's length (the
|
|
14
|
+
* bare-integer uniform shorthand carries no sum constraint, like the
|
|
15
|
+
* regular grid);
|
|
11
16
|
* - a `transpose` codec's `order` is a permutation of `0..n-1`, with one
|
|
12
17
|
* entry per array dimension;
|
|
13
18
|
* - a `sharding_indexed` codec's inner `chunk_shape` matches the array's
|
|
14
|
-
* dimensionality and evenly divides
|
|
15
|
-
*
|
|
16
|
-
* chunk when sharding
|
|
19
|
+
* dimensionality and evenly divides every chunk it shards — the grid's
|
|
20
|
+
* chunk at the top level (each distinct per-dimension size, for
|
|
21
|
+
* rectilinear grids), the parent shard's inner chunk when sharding
|
|
22
|
+
* nests;
|
|
17
23
|
* - `fill_value` has a JSON shape permitted for the named core data type
|
|
18
24
|
* (booleans for `bool`, ranged integers for the int types, numbers /
|
|
19
25
|
* the "NaN"-family sentinels / width-checked "0x…" strings for the
|
|
@@ -140,14 +146,16 @@ function isPermutation(order: number[]): boolean {
|
|
|
140
146
|
|
|
141
147
|
/**
|
|
142
148
|
* Walk one codec pipeline. `dims` is the array dimensionality at this point
|
|
143
|
-
* (undefined when unknowable); `
|
|
144
|
-
* this pipeline
|
|
149
|
+
* (undefined when unknowable); `outerChunkSizes` holds the distinct chunk
|
|
150
|
+
* lengths per dimension this pipeline may encode — what a sharding codec's
|
|
151
|
+
* inner chunks must divide. A regular grid contributes one size per
|
|
152
|
+
* dimension; a rectilinear grid may contribute several.
|
|
145
153
|
*/
|
|
146
154
|
function pipelineIssues(
|
|
147
155
|
pipeline: unknown[],
|
|
148
156
|
path: Path,
|
|
149
157
|
dims: number | undefined,
|
|
150
|
-
|
|
158
|
+
outerChunkSizes: number[][] | undefined,
|
|
151
159
|
): PathedIssue[] {
|
|
152
160
|
const issues: PathedIssue[] = [];
|
|
153
161
|
pipeline.forEach((entry, index) => {
|
|
@@ -183,21 +191,37 @@ function pipelineIssues(
|
|
|
183
191
|
kind: "invalid_value",
|
|
184
192
|
});
|
|
185
193
|
} else if (
|
|
186
|
-
|
|
187
|
-
chunkShape.length ===
|
|
188
|
-
chunkShape.every((length) => length > 0)
|
|
189
|
-
outerChunkShape.some((outer, axis) => outer % (chunkShape[axis] as number) !== 0)
|
|
194
|
+
outerChunkSizes !== undefined &&
|
|
195
|
+
chunkShape.length === outerChunkSizes.length &&
|
|
196
|
+
chunkShape.every((length) => length > 0)
|
|
190
197
|
) {
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
198
|
+
let violation: { axis: number; size: number } | undefined;
|
|
199
|
+
outerChunkSizes.forEach((sizes, axis) => {
|
|
200
|
+
if (violation !== undefined) return;
|
|
201
|
+
const bad = sizes.find((size) => size % (chunkShape[axis] as number) !== 0);
|
|
202
|
+
if (bad !== undefined) violation = { axis, size: bad };
|
|
195
203
|
});
|
|
204
|
+
if (violation !== undefined) {
|
|
205
|
+
const { axis, size } = violation;
|
|
206
|
+
const uniform = outerChunkSizes.every((sizes) => sizes.length === 1);
|
|
207
|
+
issues.push({
|
|
208
|
+
path: chunkShapePath,
|
|
209
|
+
message: uniform
|
|
210
|
+
? `expected ${JSON.stringify(chunkShape)} to evenly divide the outer chunk shape ${JSON.stringify(outerChunkSizes.map((sizes) => sizes[0]))}`
|
|
211
|
+
: `expected ${JSON.stringify(chunkShape)} to evenly divide every chunk size of the grid (dimension ${axis} has chunk size ${size})`,
|
|
212
|
+
kind: "invalid_value",
|
|
213
|
+
});
|
|
214
|
+
}
|
|
196
215
|
}
|
|
197
216
|
const inner = configuration?.["codecs"];
|
|
198
217
|
if (Array.isArray(inner)) {
|
|
199
218
|
issues.push(
|
|
200
|
-
...pipelineIssues(
|
|
219
|
+
...pipelineIssues(
|
|
220
|
+
inner,
|
|
221
|
+
[...path, index, "configuration", "codecs"],
|
|
222
|
+
dims,
|
|
223
|
+
chunkShape.map((length) => [length]),
|
|
224
|
+
),
|
|
201
225
|
);
|
|
202
226
|
}
|
|
203
227
|
// index_codecs encode the shard index, whose shape differs from the
|
|
@@ -220,18 +244,67 @@ export function validateArraySemanticsV3(value: unknown): ErrorTree {
|
|
|
220
244
|
const dims = shape?.length;
|
|
221
245
|
|
|
222
246
|
const grid = fieldParts(value["chunk_grid"]);
|
|
223
|
-
let
|
|
247
|
+
let chunkSizes: number[][] | undefined;
|
|
224
248
|
if (grid?.name === "regular") {
|
|
225
249
|
const configured = grid.configuration?.["chunk_shape"];
|
|
226
250
|
if (isIntArray(configured)) {
|
|
227
|
-
chunkShape = configured;
|
|
228
251
|
if (dims !== undefined && configured.length !== dims) {
|
|
229
252
|
issues.push({
|
|
230
253
|
path: ["chunk_grid", "configuration", "chunk_shape"],
|
|
231
254
|
message: `expected one length per dimension of shape (${dims})`,
|
|
232
255
|
kind: "invalid_value",
|
|
233
256
|
});
|
|
234
|
-
|
|
257
|
+
// wrong arity: unusable as division context
|
|
258
|
+
} else {
|
|
259
|
+
chunkSizes = configured.map((length) => [length]);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
} else if (grid?.name === "rectilinear") {
|
|
263
|
+
const specs = grid.configuration?.["chunk_shapes"];
|
|
264
|
+
if (Array.isArray(specs)) {
|
|
265
|
+
if (dims !== undefined && specs.length !== dims) {
|
|
266
|
+
issues.push({
|
|
267
|
+
path: ["chunk_grid", "configuration", "chunk_shapes"],
|
|
268
|
+
message: `expected one entry per dimension of shape (${dims})`,
|
|
269
|
+
kind: "invalid_value",
|
|
270
|
+
});
|
|
271
|
+
} else {
|
|
272
|
+
const perDim: (number[] | undefined)[] = specs.map((spec, dim) => {
|
|
273
|
+
// Bare integer: uniform shorthand, no sum constraint (edge chunks
|
|
274
|
+
// are permitted, as with the regular grid).
|
|
275
|
+
if (Number.isInteger(spec)) return spec as number > 0 ? [spec as number] : undefined;
|
|
276
|
+
if (!Array.isArray(spec)) return undefined; // registry schema's problem
|
|
277
|
+
const sizes = new Set<number>();
|
|
278
|
+
let total = 0;
|
|
279
|
+
for (const entry of spec) {
|
|
280
|
+
if (Number.isInteger(entry)) {
|
|
281
|
+
sizes.add(entry as number);
|
|
282
|
+
total += entry as number;
|
|
283
|
+
} else if (
|
|
284
|
+
Array.isArray(entry) &&
|
|
285
|
+
entry.length === 2 &&
|
|
286
|
+
Number.isInteger(entry[0]) &&
|
|
287
|
+
Number.isInteger(entry[1])
|
|
288
|
+
) {
|
|
289
|
+
sizes.add(entry[0] as number);
|
|
290
|
+
total += (entry[0] as number) * (entry[1] as number);
|
|
291
|
+
} else {
|
|
292
|
+
return undefined; // malformed entry: registry schema's problem
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
const extent = shape?.[dim];
|
|
296
|
+
if (extent !== undefined && total !== extent) {
|
|
297
|
+
issues.push({
|
|
298
|
+
path: ["chunk_grid", "configuration", "chunk_shapes", dim],
|
|
299
|
+
message: `expected chunk sizes summing to ${extent} along dimension ${dim}, got ${total}`,
|
|
300
|
+
kind: "invalid_value",
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
return [...sizes];
|
|
304
|
+
});
|
|
305
|
+
if (perDim.every((sizes) => sizes !== undefined)) {
|
|
306
|
+
chunkSizes = perDim as number[][];
|
|
307
|
+
}
|
|
235
308
|
}
|
|
236
309
|
}
|
|
237
310
|
}
|
|
@@ -246,7 +319,7 @@ export function validateArraySemanticsV3(value: unknown): ErrorTree {
|
|
|
246
319
|
|
|
247
320
|
const codecs = value["codecs"];
|
|
248
321
|
if (Array.isArray(codecs)) {
|
|
249
|
-
issues.push(...pipelineIssues(codecs, ["codecs"], dims,
|
|
322
|
+
issues.push(...pipelineIssues(codecs, ["codecs"], dims, chunkSizes));
|
|
250
323
|
}
|
|
251
324
|
return treeOf(issues);
|
|
252
325
|
}
|