zarr-metadata 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -7
- package/dist/chunk-grid/index.d.ts +27 -0
- package/dist/chunk-grid/index.d.ts.map +1 -0
- package/dist/chunk-grid/index.js +21 -0
- package/dist/chunk-grid/index.js.map +1 -0
- package/dist/chunk-grid/rectilinear.d.ts +14 -0
- package/dist/chunk-grid/rectilinear.d.ts.map +1 -0
- package/dist/{chunk-grids.js → chunk-grid/rectilinear.js} +3 -60
- package/dist/chunk-grid/rectilinear.js.map +1 -0
- package/dist/chunk-grid/regular.d.ts +7 -0
- package/dist/chunk-grid/regular.d.ts.map +1 -0
- package/dist/chunk-grid/regular.js +36 -0
- package/dist/chunk-grid/regular.js.map +1 -0
- package/dist/codec/index.d.ts +34 -0
- package/dist/codec/index.d.ts.map +1 -0
- package/dist/codec/index.js +35 -0
- package/dist/codec/index.js.map +1 -0
- package/dist/codec/sharding-indexed.d.ts +26 -0
- package/dist/codec/sharding-indexed.d.ts.map +1 -0
- package/dist/codec/sharding-indexed.js +80 -0
- package/dist/codec/sharding-indexed.js.map +1 -0
- package/dist/codec/transpose.d.ts +19 -0
- package/dist/codec/transpose.d.ts.map +1 -0
- package/dist/codec/transpose.js +66 -0
- package/dist/codec/transpose.js.map +1 -0
- package/dist/data-type/bool.d.ts +4 -0
- package/dist/data-type/bool.d.ts.map +1 -0
- package/dist/data-type/bool.js +7 -0
- package/dist/data-type/bool.js.map +1 -0
- package/dist/data-type/bytes.d.ts +11 -0
- package/dist/data-type/bytes.d.ts.map +1 -0
- package/dist/data-type/bytes.js +18 -0
- package/dist/data-type/bytes.js.map +1 -0
- package/dist/data-type/complex128.d.ts +3 -0
- package/dist/data-type/complex128.d.ts.map +1 -0
- package/dist/data-type/complex128.js +4 -0
- package/dist/data-type/complex128.js.map +1 -0
- package/dist/data-type/complex64.d.ts +3 -0
- package/dist/data-type/complex64.d.ts.map +1 -0
- package/dist/data-type/complex64.js +4 -0
- package/dist/data-type/complex64.js.map +1 -0
- package/dist/data-type/descriptor.d.ts +44 -0
- package/dist/data-type/descriptor.d.ts.map +1 -0
- package/dist/data-type/descriptor.js +57 -0
- package/dist/data-type/descriptor.js.map +1 -0
- package/dist/data-type/float16.d.ts +3 -0
- package/dist/data-type/float16.d.ts.map +1 -0
- package/dist/data-type/float16.js +4 -0
- package/dist/data-type/float16.js.map +1 -0
- package/dist/data-type/float32.d.ts +3 -0
- package/dist/data-type/float32.d.ts.map +1 -0
- package/dist/data-type/float32.js +4 -0
- package/dist/data-type/float32.js.map +1 -0
- package/dist/data-type/float64.d.ts +3 -0
- package/dist/data-type/float64.d.ts.map +1 -0
- package/dist/data-type/float64.js +4 -0
- package/dist/data-type/float64.js.map +1 -0
- package/dist/data-type/index.d.ts +23 -0
- package/dist/data-type/index.d.ts.map +1 -0
- package/dist/data-type/index.js +92 -0
- package/dist/data-type/index.js.map +1 -0
- package/dist/data-type/int16.d.ts +3 -0
- package/dist/data-type/int16.d.ts.map +1 -0
- package/dist/data-type/int16.js +4 -0
- package/dist/data-type/int16.js.map +1 -0
- package/dist/data-type/int32.d.ts +3 -0
- package/dist/data-type/int32.d.ts.map +1 -0
- package/dist/data-type/int32.js +4 -0
- package/dist/data-type/int32.js.map +1 -0
- package/dist/data-type/int64.d.ts +3 -0
- package/dist/data-type/int64.d.ts.map +1 -0
- package/dist/data-type/int64.js +4 -0
- package/dist/data-type/int64.js.map +1 -0
- package/dist/data-type/int8.d.ts +3 -0
- package/dist/data-type/int8.d.ts.map +1 -0
- package/dist/data-type/int8.js +4 -0
- package/dist/data-type/int8.js.map +1 -0
- package/dist/data-type/numpy-datetime64.d.ts +18 -0
- package/dist/data-type/numpy-datetime64.d.ts.map +1 -0
- package/dist/data-type/numpy-datetime64.js +14 -0
- package/dist/data-type/numpy-datetime64.js.map +1 -0
- package/dist/data-type/numpy-timedelta64.d.ts +4 -0
- package/dist/data-type/numpy-timedelta64.d.ts.map +1 -0
- package/dist/data-type/numpy-timedelta64.js +4 -0
- package/dist/data-type/numpy-timedelta64.js.map +1 -0
- package/dist/data-type/raw.d.ts +10 -0
- package/dist/data-type/raw.d.ts.map +1 -0
- package/dist/data-type/raw.js +26 -0
- package/dist/data-type/raw.js.map +1 -0
- package/dist/data-type/string.d.ts +6 -0
- package/dist/data-type/string.d.ts.map +1 -0
- package/dist/data-type/string.js +7 -0
- package/dist/data-type/string.js.map +1 -0
- package/dist/data-type/struct.d.ts +17 -0
- package/dist/data-type/struct.d.ts.map +1 -0
- package/dist/data-type/struct.js +44 -0
- package/dist/data-type/struct.js.map +1 -0
- package/dist/data-type/uint16.d.ts +3 -0
- package/dist/data-type/uint16.d.ts.map +1 -0
- package/dist/data-type/uint16.js +4 -0
- package/dist/data-type/uint16.js.map +1 -0
- package/dist/data-type/uint32.d.ts +3 -0
- package/dist/data-type/uint32.d.ts.map +1 -0
- package/dist/data-type/uint32.js +4 -0
- package/dist/data-type/uint32.js.map +1 -0
- package/dist/data-type/uint64.d.ts +3 -0
- package/dist/data-type/uint64.d.ts.map +1 -0
- package/dist/data-type/uint64.js +4 -0
- package/dist/data-type/uint64.js.map +1 -0
- package/dist/data-type/uint8.d.ts +3 -0
- package/dist/data-type/uint8.d.ts.map +1 -0
- package/dist/data-type/uint8.js +4 -0
- package/dist/data-type/uint8.js.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/semantics.d.ts +4 -4
- package/dist/semantics.d.ts.map +1 -1
- package/dist/semantics.js +9 -15
- package/dist/semantics.js.map +1 -1
- package/package.json +4 -2
- package/src/chunk-grid/index.ts +40 -0
- package/src/{chunk-grids.ts → chunk-grid/rectilinear.ts} +5 -75
- package/src/chunk-grid/regular.ts +42 -0
- package/src/codec/index.ts +76 -0
- package/src/codec/sharding-indexed.ts +113 -0
- package/src/codec/transpose.ts +80 -0
- package/src/data-type/bool.ts +8 -0
- package/src/data-type/bytes.ts +26 -0
- package/src/data-type/complex128.ts +4 -0
- package/src/data-type/complex64.ts +4 -0
- package/src/data-type/descriptor.ts +115 -0
- package/src/data-type/float16.ts +4 -0
- package/src/data-type/float32.ts +4 -0
- package/src/data-type/float64.ts +4 -0
- package/src/data-type/index.ts +129 -0
- package/src/data-type/int16.ts +4 -0
- package/src/data-type/int32.ts +4 -0
- package/src/data-type/int64.ts +4 -0
- package/src/data-type/int8.ts +4 -0
- package/src/data-type/numpy-datetime64.ts +33 -0
- package/src/data-type/numpy-timedelta64.ts +9 -0
- package/src/data-type/raw.ts +31 -0
- package/src/data-type/string.ts +11 -0
- package/src/data-type/struct.ts +55 -0
- package/src/data-type/uint16.ts +4 -0
- package/src/data-type/uint32.ts +4 -0
- package/src/data-type/uint64.ts +4 -0
- package/src/data-type/uint8.ts +4 -0
- package/src/index.ts +14 -3
- package/src/semantics.ts +9 -15
- package/dist/chunk-grids.d.ts +0 -40
- package/dist/chunk-grids.d.ts.map +0 -1
- package/dist/chunk-grids.js.map +0 -1
- package/dist/codecs.d.ts +0 -37
- package/dist/codecs.d.ts.map +0 -1
- package/dist/codecs.js +0 -182
- package/dist/codecs.js.map +0 -1
- package/dist/data-types.d.ts +0 -12
- package/dist/data-types.d.ts.map +0 -1
- package/dist/data-types.js +0 -70
- package/dist/data-types.js.map +0 -1
- package/src/codecs.ts +0 -214
- package/src/data-types.ts +0 -81
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/** The core `transpose` codec: syntax and semantics. */
|
|
2
|
+
import type { PathedIssue } from "../errors.js";
|
|
3
|
+
import { isIntArray, type Path } from "../guards.js";
|
|
4
|
+
import { DROPPED, type PipelineContext } from "./index.js";
|
|
5
|
+
|
|
6
|
+
/** Configuration of the core `transpose` codec. */
|
|
7
|
+
export interface TransposeCodecConfiguration {
|
|
8
|
+
order: number[];
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function isPermutation(order: number[]): boolean {
|
|
12
|
+
const seen = new Set(order);
|
|
13
|
+
return seen.size === order.length && order.every((entry) => entry >= 0 && entry < order.length);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Judge one transpose entry: it requires a configuration with `order`,
|
|
18
|
+
* which must be a permutation with one entry per array dimension. A sound
|
|
19
|
+
* transpose permutes the context's chunk sizes; an unsound one makes the
|
|
20
|
+
* downstream context unknowable.
|
|
21
|
+
*/
|
|
22
|
+
export function transposeIssues(
|
|
23
|
+
configuration: Record<string, unknown> | undefined,
|
|
24
|
+
configMissing: boolean,
|
|
25
|
+
path: Path,
|
|
26
|
+
index: number,
|
|
27
|
+
context: PipelineContext,
|
|
28
|
+
): { issues: PathedIssue[]; context: PipelineContext } {
|
|
29
|
+
const issues: PathedIssue[] = [];
|
|
30
|
+
if (configMissing) {
|
|
31
|
+
issues.push({
|
|
32
|
+
path: [...path, index],
|
|
33
|
+
message: '"transpose" requires a configuration with "order"',
|
|
34
|
+
kind: "missing_key",
|
|
35
|
+
});
|
|
36
|
+
return { issues, context: DROPPED };
|
|
37
|
+
}
|
|
38
|
+
if (configuration !== undefined && !Object.hasOwn(configuration, "order")) {
|
|
39
|
+
issues.push({
|
|
40
|
+
path: [...path, index, "configuration", "order"],
|
|
41
|
+
message: "missing required key",
|
|
42
|
+
kind: "missing_key",
|
|
43
|
+
});
|
|
44
|
+
return { issues, context: DROPPED };
|
|
45
|
+
}
|
|
46
|
+
const order = configuration?.["order"];
|
|
47
|
+
if (!isIntArray(order)) {
|
|
48
|
+
return { issues, context: DROPPED }; // shape errors are the schema layer's
|
|
49
|
+
}
|
|
50
|
+
const orderPath = [...path, index, "configuration", "order"];
|
|
51
|
+
let sound = true;
|
|
52
|
+
if (!isPermutation(order)) {
|
|
53
|
+
sound = false;
|
|
54
|
+
issues.push({
|
|
55
|
+
path: orderPath,
|
|
56
|
+
message: `expected a permutation of the integers 0..${order.length - 1}`,
|
|
57
|
+
kind: "invalid_value",
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
if (context.dims !== undefined && order.length !== context.dims) {
|
|
61
|
+
sound = false;
|
|
62
|
+
issues.push({
|
|
63
|
+
path: orderPath,
|
|
64
|
+
message: `expected one entry per array dimension (${context.dims})`,
|
|
65
|
+
kind: "invalid_value",
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
if (!sound) return { issues, context: DROPPED };
|
|
69
|
+
const sizes = context.chunkSizes;
|
|
70
|
+
return {
|
|
71
|
+
issues,
|
|
72
|
+
context: {
|
|
73
|
+
dims: context.dims,
|
|
74
|
+
chunkSizes:
|
|
75
|
+
sizes !== undefined && order.length === sizes.length
|
|
76
|
+
? order.map((axis) => sizes[axis] as number[])
|
|
77
|
+
: undefined,
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { named, simple, type DataTypeDescriptor } from "./descriptor.js";
|
|
2
|
+
|
|
3
|
+
/** The core `bool` data type. */
|
|
4
|
+
export const bool: DataTypeDescriptor = {
|
|
5
|
+
matches: named("bool"),
|
|
6
|
+
fillIssues: (fill) =>
|
|
7
|
+
typeof fill === "boolean" ? [] : simple('expected a boolean fill value for data type "bool"'),
|
|
8
|
+
};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { isByteArray, named, simple, type DataTypeDescriptor } from "./descriptor.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Fill value of the `bytes` data type: an array of integers in `[0, 255]`
|
|
5
|
+
* (one per byte) or a standard-alphabet base64 string.
|
|
6
|
+
*/
|
|
7
|
+
export type BytesFillValue = number[] | string;
|
|
8
|
+
|
|
9
|
+
const BASE64 = /^[A-Za-z0-9+/]*={0,2}$/;
|
|
10
|
+
|
|
11
|
+
/** Whether `value` is standard-alphabet base64 (padded length a multiple of 4). */
|
|
12
|
+
export function isBase64(value: string): boolean {
|
|
13
|
+
return value.length % 4 === 0 && BASE64.test(value);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** The zarr-extensions `bytes` data type (variable-length raw bytes). */
|
|
17
|
+
export const bytes: DataTypeDescriptor = {
|
|
18
|
+
matches: named("bytes"),
|
|
19
|
+
fillIssues: (fill) => {
|
|
20
|
+
if (isByteArray(fill)) return [];
|
|
21
|
+
if (typeof fill === "string" && isBase64(fill)) return [];
|
|
22
|
+
return simple(
|
|
23
|
+
'expected an array of integers in [0, 255] or a base64 string for data type "bytes"',
|
|
24
|
+
);
|
|
25
|
+
},
|
|
26
|
+
};
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The data type descriptor interface and the shared fill-value machinery
|
|
3
|
+
* the per-type modules build on. One module per supported data type;
|
|
4
|
+
* `index.ts` assembles them into the dispatch the semantic layer uses.
|
|
5
|
+
*/
|
|
6
|
+
import type { IssueKind, PathedIssue } from "../errors.js";
|
|
7
|
+
|
|
8
|
+
/** Context handed to a descriptor's fill check. */
|
|
9
|
+
export interface FillContext {
|
|
10
|
+
/** The data_type field's configuration, when present and an object. */
|
|
11
|
+
configuration: Record<string, unknown> | undefined;
|
|
12
|
+
/** Recursion depth (struct fields may nest structs). */
|
|
13
|
+
depth: number;
|
|
14
|
+
/**
|
|
15
|
+
* Recursive check for compound types: issues for `fill` judged against
|
|
16
|
+
* the data type named by `dataTypeField`, pathed relative to that fill.
|
|
17
|
+
*/
|
|
18
|
+
fillIssuesFor(dataTypeField: unknown, fill: unknown, depth: number): PathedIssue[];
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Everything the semantic layer knows about one data type. */
|
|
22
|
+
export interface DataTypeDescriptor {
|
|
23
|
+
/** Whether this module owns the given `data_type` name. */
|
|
24
|
+
matches(name: string): boolean;
|
|
25
|
+
/** Required configuration members, when the type needs a configuration. */
|
|
26
|
+
requiredConfigKeys?: readonly string[];
|
|
27
|
+
/** Name-level validity beyond ownership (e.g. r<N>'s multiple-of-8 rule). */
|
|
28
|
+
nameIssue?(name: string): string | undefined;
|
|
29
|
+
/** Fill-value issues, pathed relative to the fill value itself. */
|
|
30
|
+
fillIssues(fill: unknown, name: string, context: FillContext): PathedIssue[];
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function issue(
|
|
34
|
+
path: ReadonlyArray<string | number>,
|
|
35
|
+
message: string,
|
|
36
|
+
kind: IssueKind = "invalid_value",
|
|
37
|
+
): PathedIssue {
|
|
38
|
+
return { path, message, kind };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** A single unpathed issue — the common case for scalar types. */
|
|
42
|
+
export function simple(message: string): PathedIssue[] {
|
|
43
|
+
return [issue([], message)];
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function named(matchName: string): (name: string) => boolean {
|
|
47
|
+
return (name) => name === matchName;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// --- shared fill forms ------------------------------------------------------
|
|
51
|
+
|
|
52
|
+
/** A float fill value: a JSON number, a non-finite sentinel, or a hex string. */
|
|
53
|
+
export type FloatFillValue = number | "NaN" | "Infinity" | "-Infinity" | string;
|
|
54
|
+
|
|
55
|
+
/** A complex fill value: `[real, imaginary]`, each a float fill form. */
|
|
56
|
+
export type ComplexFillValue = [FloatFillValue, FloatFillValue];
|
|
57
|
+
|
|
58
|
+
export function isFloatFill(value: unknown, hexDigits: number): boolean {
|
|
59
|
+
if (typeof value === "number") return true;
|
|
60
|
+
if (typeof value !== "string") return false;
|
|
61
|
+
if (value === "NaN" || value === "Infinity" || value === "-Infinity") return true;
|
|
62
|
+
return new RegExp(`^0x[0-9a-fA-F]{${hexDigits}}$`).test(value);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function floatFillMessage(name: string, hexDigits: number): string {
|
|
66
|
+
return (
|
|
67
|
+
`expected a number, "NaN", "Infinity", "-Infinity", or a ` +
|
|
68
|
+
`${hexDigits}-hex-digit "0x..." string for data type ${JSON.stringify(name)}`
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function isByteArray(value: unknown): value is number[] {
|
|
73
|
+
return (
|
|
74
|
+
Array.isArray(value) &&
|
|
75
|
+
Object.keys(value).length === value.length &&
|
|
76
|
+
value.every((item) => Number.isInteger(item) && (item as number) >= 0 && (item as number) <= 255)
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// --- per-family factories ---------------------------------------------------
|
|
81
|
+
|
|
82
|
+
export function intDataType(name: string, low: number, high: number): DataTypeDescriptor {
|
|
83
|
+
return {
|
|
84
|
+
matches: named(name),
|
|
85
|
+
fillIssues: (fill) =>
|
|
86
|
+
Number.isInteger(fill) && (fill as number) >= low && (fill as number) <= high
|
|
87
|
+
? []
|
|
88
|
+
: simple(`expected an integer in [${low}, ${high}] for data type ${JSON.stringify(name)}`),
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function floatDataType(name: string, hexDigits: number): DataTypeDescriptor {
|
|
93
|
+
return {
|
|
94
|
+
matches: named(name),
|
|
95
|
+
fillIssues: (fill) =>
|
|
96
|
+
isFloatFill(fill, hexDigits) ? [] : simple(floatFillMessage(name, hexDigits)),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function complexDataType(name: string, componentHexDigits: number): DataTypeDescriptor {
|
|
101
|
+
return {
|
|
102
|
+
matches: named(name),
|
|
103
|
+
fillIssues: (fill) => {
|
|
104
|
+
const ok =
|
|
105
|
+
Array.isArray(fill) &&
|
|
106
|
+
fill.length === 2 &&
|
|
107
|
+
fill.every((part) => isFloatFill(part, componentHexDigits));
|
|
108
|
+
return ok
|
|
109
|
+
? []
|
|
110
|
+
: simple(
|
|
111
|
+
`expected a two-element [real, imaginary] array for data type ${JSON.stringify(name)}`,
|
|
112
|
+
);
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The data types this package interprets, one module per type, assembled
|
|
3
|
+
* into the dispatch the semantic layer uses: the core scalars, the r<N>
|
|
4
|
+
* raw-bits family, and the established zarr-extensions conventions
|
|
5
|
+
* (string, bytes, numpy.datetime64/timedelta64, struct). Unrecognized
|
|
6
|
+
* names yield no verdicts — the name space is open — and other types'
|
|
7
|
+
* required fields remain registry-schema facts.
|
|
8
|
+
*/
|
|
9
|
+
import type { PathedIssue } from "../errors.js";
|
|
10
|
+
import { configurationMissing, fieldParts } from "../guards.js";
|
|
11
|
+
import { bool } from "./bool.js";
|
|
12
|
+
import { bytes } from "./bytes.js";
|
|
13
|
+
import { complex64 } from "./complex64.js";
|
|
14
|
+
import { complex128 } from "./complex128.js";
|
|
15
|
+
import { issue, type DataTypeDescriptor, type FillContext } from "./descriptor.js";
|
|
16
|
+
import { float16 } from "./float16.js";
|
|
17
|
+
import { float32 } from "./float32.js";
|
|
18
|
+
import { float64 } from "./float64.js";
|
|
19
|
+
import { int8 } from "./int8.js";
|
|
20
|
+
import { int16 } from "./int16.js";
|
|
21
|
+
import { int32 } from "./int32.js";
|
|
22
|
+
import { int64 } from "./int64.js";
|
|
23
|
+
import { numpyDatetime64 } from "./numpy-datetime64.js";
|
|
24
|
+
import { numpyTimedelta64 } from "./numpy-timedelta64.js";
|
|
25
|
+
import { raw } from "./raw.js";
|
|
26
|
+
import { string } from "./string.js";
|
|
27
|
+
import { struct } from "./struct.js";
|
|
28
|
+
import { uint8 } from "./uint8.js";
|
|
29
|
+
import { uint16 } from "./uint16.js";
|
|
30
|
+
import { uint32 } from "./uint32.js";
|
|
31
|
+
import { uint64 } from "./uint64.js";
|
|
32
|
+
|
|
33
|
+
export type {
|
|
34
|
+
ComplexFillValue,
|
|
35
|
+
DataTypeDescriptor,
|
|
36
|
+
FillContext,
|
|
37
|
+
FloatFillValue,
|
|
38
|
+
} from "./descriptor.js";
|
|
39
|
+
export type { BytesFillValue } from "./bytes.js";
|
|
40
|
+
export type {
|
|
41
|
+
NumpyDatetime64Configuration,
|
|
42
|
+
NumpyDatetime64FillValue,
|
|
43
|
+
NumpyTimeUnit,
|
|
44
|
+
} from "./numpy-datetime64.js";
|
|
45
|
+
export type { RawBytesFillValue } from "./raw.js";
|
|
46
|
+
export type { StringFillValue } from "./string.js";
|
|
47
|
+
export type { StructConfiguration, StructField } from "./struct.js";
|
|
48
|
+
|
|
49
|
+
const DESCRIPTORS: readonly DataTypeDescriptor[] = [
|
|
50
|
+
bool,
|
|
51
|
+
int8, int16, int32, int64,
|
|
52
|
+
uint8, uint16, uint32, uint64,
|
|
53
|
+
float16, float32, float64,
|
|
54
|
+
complex64, complex128,
|
|
55
|
+
raw,
|
|
56
|
+
string, bytes,
|
|
57
|
+
numpyDatetime64, numpyTimedelta64,
|
|
58
|
+
struct,
|
|
59
|
+
];
|
|
60
|
+
|
|
61
|
+
function descriptorFor(name: string): DataTypeDescriptor | undefined {
|
|
62
|
+
return DESCRIPTORS.find((descriptor) => descriptor.matches(name));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// struct fields may nest structs; matches the structural layer's cap.
|
|
66
|
+
const MAX_FILL_DEPTH = 64;
|
|
67
|
+
|
|
68
|
+
function fillIssuesFor(dataTypeField: unknown, fill: unknown, depth: number): PathedIssue[] {
|
|
69
|
+
if (depth >= MAX_FILL_DEPTH) return [];
|
|
70
|
+
const parts = fieldParts(dataTypeField);
|
|
71
|
+
if (parts === undefined) return [];
|
|
72
|
+
const descriptor = descriptorFor(parts.name);
|
|
73
|
+
if (descriptor === undefined) return []; // unrecognized: no verdict
|
|
74
|
+
const context: FillContext = { configuration: parts.configuration, depth, fillIssuesFor };
|
|
75
|
+
return descriptor.fillIssues(fill, parts.name, context);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Interpret a document's `data_type` field against its `fill_value`:
|
|
80
|
+
* name-level rules, required configuration members, and the fill's JSON
|
|
81
|
+
* shape (recursively, for struct). Issues are pathed from the document
|
|
82
|
+
* root (`data_type` / `fill_value`).
|
|
83
|
+
*/
|
|
84
|
+
export function dataTypeVerdict(
|
|
85
|
+
rawField: unknown,
|
|
86
|
+
fillPresent: boolean,
|
|
87
|
+
fill: unknown,
|
|
88
|
+
): PathedIssue[] {
|
|
89
|
+
const parts = fieldParts(rawField);
|
|
90
|
+
if (parts === undefined) return [];
|
|
91
|
+
const descriptor = descriptorFor(parts.name);
|
|
92
|
+
if (descriptor === undefined) return [];
|
|
93
|
+
const issues: PathedIssue[] = [];
|
|
94
|
+
const nameProblem = descriptor.nameIssue?.(parts.name);
|
|
95
|
+
if (nameProblem !== undefined) {
|
|
96
|
+
issues.push(issue(["data_type"], nameProblem));
|
|
97
|
+
}
|
|
98
|
+
const required = descriptor.requiredConfigKeys;
|
|
99
|
+
if (required !== undefined) {
|
|
100
|
+
if (configurationMissing(rawField)) {
|
|
101
|
+
issues.push(
|
|
102
|
+
issue(
|
|
103
|
+
["data_type"],
|
|
104
|
+
`${JSON.stringify(parts.name)} requires a configuration with ${required
|
|
105
|
+
.map((key) => JSON.stringify(key))
|
|
106
|
+
.join(" and ")}`,
|
|
107
|
+
"missing_key",
|
|
108
|
+
),
|
|
109
|
+
);
|
|
110
|
+
} else if (parts.configuration !== undefined) {
|
|
111
|
+
for (const key of required) {
|
|
112
|
+
if (!Object.hasOwn(parts.configuration, key)) {
|
|
113
|
+
issues.push(
|
|
114
|
+
issue(["data_type", "configuration", key], "missing required key", "missing_key"),
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (fillPresent) {
|
|
121
|
+
issues.push(
|
|
122
|
+
...fillIssuesFor(rawField, fill, 0).map((inner) => ({
|
|
123
|
+
...inner,
|
|
124
|
+
path: ["fill_value", ...inner.path],
|
|
125
|
+
})),
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
return issues;
|
|
129
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { named, simple, type DataTypeDescriptor } from "./descriptor.js";
|
|
2
|
+
|
|
3
|
+
/** Time unit codes used by numpy.datetime64 / numpy.timedelta64. */
|
|
4
|
+
export type NumpyTimeUnit =
|
|
5
|
+
| "Y" | "M" | "W" | "D" | "h" | "m" | "s"
|
|
6
|
+
| "ms" | "us" | "μs" | "ns" | "ps" | "fs" | "as" | "generic";
|
|
7
|
+
|
|
8
|
+
/** Configuration of the numpy temporal data types. */
|
|
9
|
+
export interface NumpyDatetime64Configuration {
|
|
10
|
+
unit: NumpyTimeUnit;
|
|
11
|
+
scale_factor: number;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Fill value of the numpy temporal data types: an integer count of
|
|
16
|
+
* `unit * scale_factor` since the epoch, or the `"NaT"` sentinel.
|
|
17
|
+
*/
|
|
18
|
+
export type NumpyDatetime64FillValue = number | "NaT";
|
|
19
|
+
|
|
20
|
+
/** Descriptor factory shared by numpy.datetime64 and numpy.timedelta64. */
|
|
21
|
+
export function numpyTemporalDataType(name: string): DataTypeDescriptor {
|
|
22
|
+
return {
|
|
23
|
+
matches: named(name),
|
|
24
|
+
requiredConfigKeys: ["unit", "scale_factor"],
|
|
25
|
+
fillIssues: (fill) =>
|
|
26
|
+
Number.isInteger(fill) || fill === "NaT"
|
|
27
|
+
? []
|
|
28
|
+
: simple(`expected an integer or "NaT" for data type ${JSON.stringify(name)}`),
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** The zarr-extensions `numpy.datetime64` data type. */
|
|
33
|
+
export const numpyDatetime64 = numpyTemporalDataType("numpy.datetime64");
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { numpyTemporalDataType } from "./numpy-datetime64.js";
|
|
2
|
+
|
|
3
|
+
export type {
|
|
4
|
+
NumpyDatetime64Configuration as NumpyTimedelta64Configuration,
|
|
5
|
+
NumpyDatetime64FillValue as NumpyTimedelta64FillValue,
|
|
6
|
+
} from "./numpy-datetime64.js";
|
|
7
|
+
|
|
8
|
+
/** The zarr-extensions `numpy.timedelta64` data type. */
|
|
9
|
+
export const numpyTimedelta64 = numpyTemporalDataType("numpy.timedelta64");
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { isByteArray, simple, type DataTypeDescriptor } from "./descriptor.js";
|
|
2
|
+
|
|
3
|
+
/** Fill value of the `r<N>` raw-bits data types: one integer per byte. */
|
|
4
|
+
export type RawBytesFillValue = number[];
|
|
5
|
+
|
|
6
|
+
const RAW = /^r([1-9][0-9]*)$/;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The core `r<N>` raw-bits data type family. The name itself carries a
|
|
10
|
+
* rule — N must be a positive multiple of 8 — and the fill value must
|
|
11
|
+
* hold exactly N/8 bytes.
|
|
12
|
+
*/
|
|
13
|
+
export const raw: DataTypeDescriptor = {
|
|
14
|
+
matches: (name) => RAW.test(name),
|
|
15
|
+
nameIssue: (name) => {
|
|
16
|
+
const bits = Number((RAW.exec(name) as RegExpExecArray)[1]);
|
|
17
|
+
return bits % 8 === 0
|
|
18
|
+
? undefined
|
|
19
|
+
: `expected "r<N>" with N a positive multiple of 8, got ${JSON.stringify(name)}`;
|
|
20
|
+
},
|
|
21
|
+
fillIssues: (fill, name) => {
|
|
22
|
+
const bits = Number((RAW.exec(name) as RegExpExecArray)[1]);
|
|
23
|
+
if (bits % 8 !== 0) return []; // the name issue already covers this document
|
|
24
|
+
const expectedBytes = bits / 8;
|
|
25
|
+
return isByteArray(fill) && fill.length === expectedBytes
|
|
26
|
+
? []
|
|
27
|
+
: simple(
|
|
28
|
+
`expected an array of ${expectedBytes} integers in [0, 255] for data type ${JSON.stringify(name)}`,
|
|
29
|
+
);
|
|
30
|
+
},
|
|
31
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { named, simple, type DataTypeDescriptor } from "./descriptor.js";
|
|
2
|
+
|
|
3
|
+
/** Fill value of the `string` data type: a JSON unicode string. */
|
|
4
|
+
export type StringFillValue = string;
|
|
5
|
+
|
|
6
|
+
/** The zarr-extensions `string` data type (variable-length UTF-8). */
|
|
7
|
+
export const string: DataTypeDescriptor = {
|
|
8
|
+
matches: named("string"),
|
|
9
|
+
fillIssues: (fill) =>
|
|
10
|
+
typeof fill === "string" ? [] : simple('expected a string fill value for data type "string"'),
|
|
11
|
+
};
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { isPlainObject } from "../guards.js";
|
|
2
|
+
import { issue, named, simple, type DataTypeDescriptor } from "./descriptor.js";
|
|
3
|
+
|
|
4
|
+
/** One field of a `struct` data type; `data_type` recurses (structs may nest). */
|
|
5
|
+
export interface StructField {
|
|
6
|
+
name: string;
|
|
7
|
+
data_type: unknown;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/** Configuration of the `struct` data type. */
|
|
11
|
+
export interface StructConfiguration {
|
|
12
|
+
fields: StructField[];
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The zarr-extensions `struct` data type (heterogeneous record). Its fill
|
|
17
|
+
* value is a JSON object mapping every field name to that field's fill
|
|
18
|
+
* value, each judged recursively against the field's own data type.
|
|
19
|
+
*/
|
|
20
|
+
export const struct: DataTypeDescriptor = {
|
|
21
|
+
matches: named("struct"),
|
|
22
|
+
requiredConfigKeys: ["fields"],
|
|
23
|
+
fillIssues: (fill, _name, context) => {
|
|
24
|
+
if (!isPlainObject(fill)) {
|
|
25
|
+
return simple('expected an object mapping field names to fill values for data type "struct"');
|
|
26
|
+
}
|
|
27
|
+
const fields = context.configuration?.["fields"];
|
|
28
|
+
if (!Array.isArray(fields)) return []; // config shape is the schema layer's problem
|
|
29
|
+
const declared = new Map<string, unknown>();
|
|
30
|
+
for (const field of fields) {
|
|
31
|
+
if (isPlainObject(field) && typeof field["name"] === "string") {
|
|
32
|
+
declared.set(field["name"], field["data_type"]);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
if (declared.size !== fields.length) return []; // malformed fields: schema layer's problem
|
|
36
|
+
const issues = [];
|
|
37
|
+
for (const [fieldName, dataType] of declared) {
|
|
38
|
+
if (!Object.hasOwn(fill, fieldName)) {
|
|
39
|
+
issues.push(issue([fieldName], "missing required key", "missing_key"));
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
issues.push(
|
|
43
|
+
...context
|
|
44
|
+
.fillIssuesFor(dataType, fill[fieldName], context.depth + 1)
|
|
45
|
+
.map((inner) => ({ ...inner, path: [fieldName, ...inner.path] })),
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
for (const key of Object.keys(fill)) {
|
|
49
|
+
if (!declared.has(key)) {
|
|
50
|
+
issues.push(issue([key], "unexpected member (no such struct field)"));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return issues;
|
|
54
|
+
},
|
|
55
|
+
};
|
package/src/index.ts
CHANGED
|
@@ -25,12 +25,23 @@ export type {
|
|
|
25
25
|
RectilinearChunkGridConfiguration,
|
|
26
26
|
RectilinearDimSpec,
|
|
27
27
|
RegularChunkGridConfiguration,
|
|
28
|
-
} from "./chunk-
|
|
28
|
+
} from "./chunk-grid/index.js";
|
|
29
29
|
export type {
|
|
30
30
|
ShardingIndexedCodecConfiguration,
|
|
31
31
|
TransposeCodecConfiguration,
|
|
32
|
-
} from "./
|
|
33
|
-
export type {
|
|
32
|
+
} from "./codec/index.js";
|
|
33
|
+
export type {
|
|
34
|
+
BytesFillValue,
|
|
35
|
+
ComplexFillValue,
|
|
36
|
+
FloatFillValue,
|
|
37
|
+
NumpyDatetime64Configuration,
|
|
38
|
+
NumpyDatetime64FillValue,
|
|
39
|
+
NumpyTimeUnit,
|
|
40
|
+
RawBytesFillValue,
|
|
41
|
+
StringFillValue,
|
|
42
|
+
StructConfiguration,
|
|
43
|
+
StructField,
|
|
44
|
+
} from "./data-type/index.js";
|
|
34
45
|
export * from "./schemas.js";
|
|
35
46
|
export * from "./semantics.js";
|
|
36
47
|
export type { StandardSchemaV1 } from "./standard-schema.js";
|
package/src/semantics.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Semantic (cross-field) validation for v3 array metadata: the
|
|
3
3
|
* orchestrator. The rules themselves live with their content —
|
|
4
|
-
* `chunk-
|
|
5
|
-
* (
|
|
6
|
-
*
|
|
7
|
-
* and the zarr-extensions registry are organized. This module only walks
|
|
4
|
+
* `chunk-grid/`, `codec/`, and `data-type/` hold one module per
|
|
5
|
+
* interpreted name (grids, codecs, data types), each with its syntax
|
|
6
|
+
* (types) and semantics (procedures), mirroring how the spec, the Python
|
|
7
|
+
* reference's type layout, and the zarr-extensions registry are organized. This module only walks
|
|
8
8
|
* documents: extract shape, ask each content module for its verdicts,
|
|
9
9
|
* thread the chunk-size context into codec pipelines, and descend into a
|
|
10
10
|
* group's inline consolidated entries.
|
|
@@ -18,11 +18,11 @@
|
|
|
18
18
|
* modules.
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
|
-
import { chunkGridVerdict } from "./chunk-
|
|
22
|
-
import { pipelineIssues } from "./
|
|
23
|
-
import {
|
|
21
|
+
import { chunkGridVerdict } from "./chunk-grid/index.js";
|
|
22
|
+
import { pipelineIssues } from "./codec/index.js";
|
|
23
|
+
import { dataTypeVerdict } from "./data-type/index.js";
|
|
24
24
|
import { treeOf, type ErrorTree, type PathedIssue } from "./errors.js";
|
|
25
|
-
import {
|
|
25
|
+
import { isIntArray, isPlainObject } from "./guards.js";
|
|
26
26
|
|
|
27
27
|
function arraySemanticsIssues(value: unknown): PathedIssue[] {
|
|
28
28
|
if (!isPlainObject(value) || value["node_type"] !== "array") return [];
|
|
@@ -34,13 +34,7 @@ function arraySemanticsIssues(value: unknown): PathedIssue[] {
|
|
|
34
34
|
...grid.issues.map((issue) => ({ ...issue, path: ["chunk_grid", ...issue.path] })),
|
|
35
35
|
);
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
if (dataType !== undefined && "fill_value" in value) {
|
|
39
|
-
const message = fillValueIssue(dataType.name, value["fill_value"]);
|
|
40
|
-
if (message !== undefined) {
|
|
41
|
-
issues.push({ path: ["fill_value"], message, kind: "invalid_value" });
|
|
42
|
-
}
|
|
43
|
-
}
|
|
37
|
+
issues.push(...dataTypeVerdict(value["data_type"], "fill_value" in value, value["fill_value"]));
|
|
44
38
|
|
|
45
39
|
const codecs = value["codecs"];
|
|
46
40
|
if (Array.isArray(codecs)) {
|