zarr-metadata 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -6
- package/dist/chunk-grid/index.d.ts +27 -0
- package/dist/chunk-grid/index.d.ts.map +1 -0
- package/dist/chunk-grid/index.js +21 -0
- package/dist/chunk-grid/index.js.map +1 -0
- package/dist/chunk-grid/rectilinear.d.ts +14 -0
- package/dist/chunk-grid/rectilinear.d.ts.map +1 -0
- package/dist/chunk-grid/rectilinear.js +76 -0
- package/dist/chunk-grid/rectilinear.js.map +1 -0
- package/dist/chunk-grid/regular.d.ts +7 -0
- package/dist/chunk-grid/regular.d.ts.map +1 -0
- package/dist/chunk-grid/regular.js +36 -0
- package/dist/chunk-grid/regular.js.map +1 -0
- package/dist/codec/index.d.ts +34 -0
- package/dist/codec/index.d.ts.map +1 -0
- package/dist/codec/index.js +35 -0
- package/dist/codec/index.js.map +1 -0
- package/dist/codec/sharding-indexed.d.ts +26 -0
- package/dist/codec/sharding-indexed.d.ts.map +1 -0
- package/dist/codec/sharding-indexed.js +80 -0
- package/dist/codec/sharding-indexed.js.map +1 -0
- package/dist/codec/transpose.d.ts +19 -0
- package/dist/codec/transpose.d.ts.map +1 -0
- package/dist/codec/transpose.js +66 -0
- package/dist/codec/transpose.js.map +1 -0
- package/dist/data-type/bool.d.ts +4 -0
- package/dist/data-type/bool.d.ts.map +1 -0
- package/dist/data-type/bool.js +7 -0
- package/dist/data-type/bool.js.map +1 -0
- package/dist/data-type/bytes.d.ts +11 -0
- package/dist/data-type/bytes.d.ts.map +1 -0
- package/dist/data-type/bytes.js +18 -0
- package/dist/data-type/bytes.js.map +1 -0
- package/dist/data-type/complex128.d.ts +3 -0
- package/dist/data-type/complex128.d.ts.map +1 -0
- package/dist/data-type/complex128.js +4 -0
- package/dist/data-type/complex128.js.map +1 -0
- package/dist/data-type/complex64.d.ts +3 -0
- package/dist/data-type/complex64.d.ts.map +1 -0
- package/dist/data-type/complex64.js +4 -0
- package/dist/data-type/complex64.js.map +1 -0
- package/dist/data-type/descriptor.d.ts +44 -0
- package/dist/data-type/descriptor.d.ts.map +1 -0
- package/dist/data-type/descriptor.js +57 -0
- package/dist/data-type/descriptor.js.map +1 -0
- package/dist/data-type/float16.d.ts +3 -0
- package/dist/data-type/float16.d.ts.map +1 -0
- package/dist/data-type/float16.js +4 -0
- package/dist/data-type/float16.js.map +1 -0
- package/dist/data-type/float32.d.ts +3 -0
- package/dist/data-type/float32.d.ts.map +1 -0
- package/dist/data-type/float32.js +4 -0
- package/dist/data-type/float32.js.map +1 -0
- package/dist/data-type/float64.d.ts +3 -0
- package/dist/data-type/float64.d.ts.map +1 -0
- package/dist/data-type/float64.js +4 -0
- package/dist/data-type/float64.js.map +1 -0
- package/dist/data-type/index.d.ts +23 -0
- package/dist/data-type/index.d.ts.map +1 -0
- package/dist/data-type/index.js +92 -0
- package/dist/data-type/index.js.map +1 -0
- package/dist/data-type/int16.d.ts +3 -0
- package/dist/data-type/int16.d.ts.map +1 -0
- package/dist/data-type/int16.js +4 -0
- package/dist/data-type/int16.js.map +1 -0
- package/dist/data-type/int32.d.ts +3 -0
- package/dist/data-type/int32.d.ts.map +1 -0
- package/dist/data-type/int32.js +4 -0
- package/dist/data-type/int32.js.map +1 -0
- package/dist/data-type/int64.d.ts +3 -0
- package/dist/data-type/int64.d.ts.map +1 -0
- package/dist/data-type/int64.js +4 -0
- package/dist/data-type/int64.js.map +1 -0
- package/dist/data-type/int8.d.ts +3 -0
- package/dist/data-type/int8.d.ts.map +1 -0
- package/dist/data-type/int8.js +4 -0
- package/dist/data-type/int8.js.map +1 -0
- package/dist/data-type/numpy-datetime64.d.ts +18 -0
- package/dist/data-type/numpy-datetime64.d.ts.map +1 -0
- package/dist/data-type/numpy-datetime64.js +14 -0
- package/dist/data-type/numpy-datetime64.js.map +1 -0
- package/dist/data-type/numpy-timedelta64.d.ts +4 -0
- package/dist/data-type/numpy-timedelta64.d.ts.map +1 -0
- package/dist/data-type/numpy-timedelta64.js +4 -0
- package/dist/data-type/numpy-timedelta64.js.map +1 -0
- package/dist/data-type/raw.d.ts +10 -0
- package/dist/data-type/raw.d.ts.map +1 -0
- package/dist/data-type/raw.js +26 -0
- package/dist/data-type/raw.js.map +1 -0
- package/dist/data-type/string.d.ts +6 -0
- package/dist/data-type/string.d.ts.map +1 -0
- package/dist/data-type/string.js +7 -0
- package/dist/data-type/string.js.map +1 -0
- package/dist/data-type/struct.d.ts +17 -0
- package/dist/data-type/struct.d.ts.map +1 -0
- package/dist/data-type/struct.js +44 -0
- package/dist/data-type/struct.js.map +1 -0
- package/dist/data-type/uint16.d.ts +3 -0
- package/dist/data-type/uint16.d.ts.map +1 -0
- package/dist/data-type/uint16.js +4 -0
- package/dist/data-type/uint16.js.map +1 -0
- package/dist/data-type/uint32.d.ts +3 -0
- package/dist/data-type/uint32.d.ts.map +1 -0
- package/dist/data-type/uint32.js +4 -0
- package/dist/data-type/uint32.js.map +1 -0
- package/dist/data-type/uint64.d.ts +3 -0
- package/dist/data-type/uint64.d.ts.map +1 -0
- package/dist/data-type/uint64.js +4 -0
- package/dist/data-type/uint64.js.map +1 -0
- package/dist/data-type/uint8.d.ts +3 -0
- package/dist/data-type/uint8.d.ts.map +1 -0
- package/dist/data-type/uint8.js +4 -0
- package/dist/data-type/uint8.js.map +1 -0
- package/dist/guards.d.ts +21 -0
- package/dist/guards.d.ts.map +1 -0
- package/dist/guards.js +39 -0
- package/dist/guards.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/semantics.d.ts +14 -42
- package/dist/semantics.d.ts.map +1 -1
- package/dist/semantics.js +22 -416
- package/dist/semantics.js.map +1 -1
- package/package.json +4 -2
- package/src/chunk-grid/index.ts +40 -0
- package/src/chunk-grid/rectilinear.ts +88 -0
- package/src/chunk-grid/regular.ts +42 -0
- package/src/codec/index.ts +76 -0
- package/src/codec/sharding-indexed.ts +113 -0
- package/src/codec/transpose.ts +80 -0
- package/src/data-type/bool.ts +8 -0
- package/src/data-type/bytes.ts +26 -0
- package/src/data-type/complex128.ts +4 -0
- package/src/data-type/complex64.ts +4 -0
- package/src/data-type/descriptor.ts +115 -0
- package/src/data-type/float16.ts +4 -0
- package/src/data-type/float32.ts +4 -0
- package/src/data-type/float64.ts +4 -0
- package/src/data-type/index.ts +129 -0
- package/src/data-type/int16.ts +4 -0
- package/src/data-type/int32.ts +4 -0
- package/src/data-type/int64.ts +4 -0
- package/src/data-type/int8.ts +4 -0
- package/src/data-type/numpy-datetime64.ts +33 -0
- package/src/data-type/numpy-timedelta64.ts +9 -0
- package/src/data-type/raw.ts +31 -0
- package/src/data-type/string.ts +11 -0
- package/src/data-type/struct.ts +55 -0
- package/src/data-type/uint16.ts +4 -0
- package/src/data-type/uint32.ts +4 -0
- package/src/data-type/uint64.ts +4 -0
- package/src/data-type/uint8.ts +4 -0
- package/src/guards.ts +48 -0
- package/src/index.ts +22 -0
- package/src/semantics.ts +24 -444
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/** The core `regular` chunk grid: syntax and semantics. */
|
|
2
|
+
import type { PathedIssue } from "../errors.js";
|
|
3
|
+
import { configurationMissing, fieldParts, isIntArray } from "../guards.js";
|
|
4
|
+
import type { ChunkGridVerdict } from "./index.js";
|
|
5
|
+
|
|
6
|
+
/** Configuration of the core `regular` chunk grid. */
|
|
7
|
+
export interface RegularChunkGridConfiguration {
|
|
8
|
+
chunk_shape: number[];
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function regularIssues(rawGrid: unknown, shape: number[] | undefined): ChunkGridVerdict {
|
|
12
|
+
const issues: PathedIssue[] = [];
|
|
13
|
+
let chunkSizes: number[][] | undefined;
|
|
14
|
+
const configuration = fieldParts(rawGrid)?.configuration;
|
|
15
|
+
if (configurationMissing(rawGrid)) {
|
|
16
|
+
issues.push({
|
|
17
|
+
path: [],
|
|
18
|
+
message: '"regular" requires a configuration with "chunk_shape"',
|
|
19
|
+
kind: "missing_key",
|
|
20
|
+
});
|
|
21
|
+
} else if (configuration !== undefined && !Object.hasOwn(configuration, "chunk_shape")) {
|
|
22
|
+
issues.push({
|
|
23
|
+
path: ["configuration", "chunk_shape"],
|
|
24
|
+
message: "missing required key",
|
|
25
|
+
kind: "missing_key",
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
const configured = configuration?.["chunk_shape"];
|
|
29
|
+
if (isIntArray(configured)) {
|
|
30
|
+
if (shape !== undefined && configured.length !== shape.length) {
|
|
31
|
+
issues.push({
|
|
32
|
+
path: ["configuration", "chunk_shape"],
|
|
33
|
+
message: `expected one length per dimension of shape (${shape.length})`,
|
|
34
|
+
kind: "invalid_value",
|
|
35
|
+
});
|
|
36
|
+
// wrong arity: unusable as division context
|
|
37
|
+
} else {
|
|
38
|
+
chunkSizes = configured.map((length) => [length]);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return { issues, chunkSizes };
|
|
42
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The codecs this package interprets, one module per codec, plus the
|
|
3
|
+
* pipeline walker that threads dimensional context between codecs
|
|
4
|
+
* (mirroring zarr-python's chunk-spec threading): a valid transpose
|
|
5
|
+
* permutes the per-dimension chunk sizes for the codecs after it, and any
|
|
6
|
+
* codec this package cannot reason about — `reshape` may change a chunk's
|
|
7
|
+
* rank, and unknown codecs may do anything — invalidates the dimensional
|
|
8
|
+
* context for the rest of the pipeline instead of letting stale
|
|
9
|
+
* array-level facts produce false verdicts.
|
|
10
|
+
*
|
|
11
|
+
* Other codecs' required fields (blosc, gzip, zstd, ...) are deliberately
|
|
12
|
+
* NOT known here: they are registry-schema facts enforced by schema-driven
|
|
13
|
+
* tooling, and a second hardcoded source of truth would drift.
|
|
14
|
+
*/
|
|
15
|
+
import type { PathedIssue } from "../errors.js";
|
|
16
|
+
import { configurationMissing, fieldParts, type Path } from "../guards.js";
|
|
17
|
+
import { shardingIndexedIssues } from "./sharding-indexed.js";
|
|
18
|
+
import { transposeIssues } from "./transpose.js";
|
|
19
|
+
|
|
20
|
+
export type { ShardingIndexedCodecConfiguration } from "./sharding-indexed.js";
|
|
21
|
+
export type { TransposeCodecConfiguration } from "./transpose.js";
|
|
22
|
+
|
|
23
|
+
/** The dimensional facts threaded between the codecs of one pipeline. */
|
|
24
|
+
export interface PipelineContext {
|
|
25
|
+
/** Array dimensionality at this point; undefined when unknowable. */
|
|
26
|
+
dims: number | undefined;
|
|
27
|
+
/**
|
|
28
|
+
* Distinct chunk lengths per dimension this pipeline may encode — what a
|
|
29
|
+
* sharding codec's inner chunks must divide. A regular grid contributes
|
|
30
|
+
* one size per dimension; a rectilinear grid may contribute several.
|
|
31
|
+
*/
|
|
32
|
+
chunkSizes: number[][] | undefined;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** The spent/unknowable context. */
|
|
36
|
+
export const DROPPED: PipelineContext = { dims: undefined, chunkSizes: undefined };
|
|
37
|
+
|
|
38
|
+
/** Walk one codec pipeline, threading the dimensional context. */
|
|
39
|
+
export function pipelineIssues(
|
|
40
|
+
pipeline: unknown[],
|
|
41
|
+
path: Path,
|
|
42
|
+
initialDims: number | undefined,
|
|
43
|
+
initialChunkSizes: number[][] | undefined,
|
|
44
|
+
): PathedIssue[] {
|
|
45
|
+
const issues: PathedIssue[] = [];
|
|
46
|
+
let context: PipelineContext = { dims: initialDims, chunkSizes: initialChunkSizes };
|
|
47
|
+
pipeline.forEach((entry, index) => {
|
|
48
|
+
const parts = fieldParts(entry);
|
|
49
|
+
if (parts === undefined) {
|
|
50
|
+
context = DROPPED; // structurally invalid entry: no further reasoning
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
const configMissing = configurationMissing(entry);
|
|
54
|
+
if (parts.name === "transpose") {
|
|
55
|
+
const result = transposeIssues(parts.configuration, configMissing, path, index, context);
|
|
56
|
+
issues.push(...result.issues);
|
|
57
|
+
context = result.context;
|
|
58
|
+
} else if (parts.name === "sharding_indexed") {
|
|
59
|
+
const result = shardingIndexedIssues(
|
|
60
|
+
parts.configuration,
|
|
61
|
+
configMissing,
|
|
62
|
+
path,
|
|
63
|
+
index,
|
|
64
|
+
context,
|
|
65
|
+
pipelineIssues,
|
|
66
|
+
);
|
|
67
|
+
issues.push(...result.issues);
|
|
68
|
+
context = result.context;
|
|
69
|
+
} else {
|
|
70
|
+
// A codec this package cannot reason about: stale array-level facts
|
|
71
|
+
// must not judge the codecs after it.
|
|
72
|
+
context = DROPPED;
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
return issues;
|
|
76
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/** The core `sharding_indexed` codec: syntax and semantics. */
|
|
2
|
+
import type { PathedIssue } from "../errors.js";
|
|
3
|
+
import { isIntArray, type Path } from "../guards.js";
|
|
4
|
+
import { DROPPED, type PipelineContext } from "./index.js";
|
|
5
|
+
|
|
6
|
+
/** Configuration of the core `sharding_indexed` codec. */
|
|
7
|
+
export interface ShardingIndexedCodecConfiguration {
|
|
8
|
+
chunk_shape: number[];
|
|
9
|
+
codecs: unknown[];
|
|
10
|
+
index_codecs: unknown[];
|
|
11
|
+
index_location?: "start" | "end";
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
type Walk = (
|
|
15
|
+
pipeline: unknown[],
|
|
16
|
+
path: Path,
|
|
17
|
+
dims: number | undefined,
|
|
18
|
+
chunkSizes: number[][] | undefined,
|
|
19
|
+
) => PathedIssue[];
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Judge one sharding entry: it requires `chunk_shape`, `codecs`, and
|
|
23
|
+
* `index_codecs`; its chunk_shape must match the array's dimensionality
|
|
24
|
+
* and evenly divide every chunk it shards. The inner pipeline is walked
|
|
25
|
+
* with the shard's own chunk context (`index_codecs` encode the shard
|
|
26
|
+
* index, whose shape differs — no dimensional context applies there), and
|
|
27
|
+
* the array -> bytes boundary spends the context for whatever follows.
|
|
28
|
+
*/
|
|
29
|
+
export function shardingIndexedIssues(
|
|
30
|
+
configuration: Record<string, unknown> | undefined,
|
|
31
|
+
configMissing: boolean,
|
|
32
|
+
path: Path,
|
|
33
|
+
index: number,
|
|
34
|
+
context: PipelineContext,
|
|
35
|
+
walk: Walk,
|
|
36
|
+
): { issues: PathedIssue[]; context: PipelineContext } {
|
|
37
|
+
const issues: PathedIssue[] = [];
|
|
38
|
+
if (configMissing) {
|
|
39
|
+
issues.push({
|
|
40
|
+
path: [...path, index],
|
|
41
|
+
message:
|
|
42
|
+
'"sharding_indexed" requires a configuration with "chunk_shape", "codecs", and "index_codecs"',
|
|
43
|
+
kind: "missing_key",
|
|
44
|
+
});
|
|
45
|
+
return { issues, context: DROPPED };
|
|
46
|
+
}
|
|
47
|
+
if (configuration !== undefined) {
|
|
48
|
+
for (const key of ["chunk_shape", "codecs", "index_codecs"]) {
|
|
49
|
+
if (!Object.hasOwn(configuration, key)) {
|
|
50
|
+
issues.push({
|
|
51
|
+
path: [...path, index, "configuration", key],
|
|
52
|
+
message: "missing required key",
|
|
53
|
+
kind: "missing_key",
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
const chunkShape = configuration?.["chunk_shape"];
|
|
59
|
+
if (!isIntArray(chunkShape)) {
|
|
60
|
+
// Still walk a present inner pipeline (entries deserve their own
|
|
61
|
+
// verdicts) before the context becomes unknowable.
|
|
62
|
+
const inner = configuration?.["codecs"];
|
|
63
|
+
if (Array.isArray(inner)) {
|
|
64
|
+
issues.push(
|
|
65
|
+
...walk(inner, [...path, index, "configuration", "codecs"], undefined, undefined),
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
return { issues, context: DROPPED };
|
|
69
|
+
}
|
|
70
|
+
const chunkShapePath = [...path, index, "configuration", "chunk_shape"];
|
|
71
|
+
if (context.dims !== undefined && chunkShape.length !== context.dims) {
|
|
72
|
+
issues.push({
|
|
73
|
+
path: chunkShapePath,
|
|
74
|
+
message: `expected one length per array dimension (${context.dims})`,
|
|
75
|
+
kind: "invalid_value",
|
|
76
|
+
});
|
|
77
|
+
} else if (
|
|
78
|
+
context.chunkSizes !== undefined &&
|
|
79
|
+
chunkShape.length === context.chunkSizes.length &&
|
|
80
|
+
chunkShape.every((length) => length > 0)
|
|
81
|
+
) {
|
|
82
|
+
let violation: { axis: number; size: number } | undefined;
|
|
83
|
+
context.chunkSizes.forEach((sizes, axis) => {
|
|
84
|
+
if (violation !== undefined) return;
|
|
85
|
+
const bad = sizes.find((size) => size % (chunkShape[axis] as number) !== 0);
|
|
86
|
+
if (bad !== undefined) violation = { axis, size: bad };
|
|
87
|
+
});
|
|
88
|
+
if (violation !== undefined) {
|
|
89
|
+
const { axis, size } = violation;
|
|
90
|
+
const sizesNow = context.chunkSizes;
|
|
91
|
+
const uniform = sizesNow.every((sizes) => sizes.length === 1);
|
|
92
|
+
issues.push({
|
|
93
|
+
path: chunkShapePath,
|
|
94
|
+
message: uniform
|
|
95
|
+
? `expected ${JSON.stringify(chunkShape)} to evenly divide the outer chunk shape ${JSON.stringify(sizesNow.map((sizes) => sizes[0]))}`
|
|
96
|
+
: `expected ${JSON.stringify(chunkShape)} to evenly divide every chunk size of the grid (dimension ${axis} has chunk size ${size})`,
|
|
97
|
+
kind: "invalid_value",
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const inner = configuration?.["codecs"];
|
|
102
|
+
if (Array.isArray(inner)) {
|
|
103
|
+
issues.push(
|
|
104
|
+
...walk(
|
|
105
|
+
inner,
|
|
106
|
+
[...path, index, "configuration", "codecs"],
|
|
107
|
+
chunkShape.length,
|
|
108
|
+
chunkShape.map((length) => [length]),
|
|
109
|
+
),
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
return { issues, context: DROPPED };
|
|
113
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/** The core `transpose` codec: syntax and semantics. */
|
|
2
|
+
import type { PathedIssue } from "../errors.js";
|
|
3
|
+
import { isIntArray, type Path } from "../guards.js";
|
|
4
|
+
import { DROPPED, type PipelineContext } from "./index.js";
|
|
5
|
+
|
|
6
|
+
/** Configuration of the core `transpose` codec. */
|
|
7
|
+
export interface TransposeCodecConfiguration {
|
|
8
|
+
order: number[];
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function isPermutation(order: number[]): boolean {
|
|
12
|
+
const seen = new Set(order);
|
|
13
|
+
return seen.size === order.length && order.every((entry) => entry >= 0 && entry < order.length);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Judge one transpose entry: it requires a configuration with `order`,
|
|
18
|
+
* which must be a permutation with one entry per array dimension. A sound
|
|
19
|
+
* transpose permutes the context's chunk sizes; an unsound one makes the
|
|
20
|
+
* downstream context unknowable.
|
|
21
|
+
*/
|
|
22
|
+
export function transposeIssues(
|
|
23
|
+
configuration: Record<string, unknown> | undefined,
|
|
24
|
+
configMissing: boolean,
|
|
25
|
+
path: Path,
|
|
26
|
+
index: number,
|
|
27
|
+
context: PipelineContext,
|
|
28
|
+
): { issues: PathedIssue[]; context: PipelineContext } {
|
|
29
|
+
const issues: PathedIssue[] = [];
|
|
30
|
+
if (configMissing) {
|
|
31
|
+
issues.push({
|
|
32
|
+
path: [...path, index],
|
|
33
|
+
message: '"transpose" requires a configuration with "order"',
|
|
34
|
+
kind: "missing_key",
|
|
35
|
+
});
|
|
36
|
+
return { issues, context: DROPPED };
|
|
37
|
+
}
|
|
38
|
+
if (configuration !== undefined && !Object.hasOwn(configuration, "order")) {
|
|
39
|
+
issues.push({
|
|
40
|
+
path: [...path, index, "configuration", "order"],
|
|
41
|
+
message: "missing required key",
|
|
42
|
+
kind: "missing_key",
|
|
43
|
+
});
|
|
44
|
+
return { issues, context: DROPPED };
|
|
45
|
+
}
|
|
46
|
+
const order = configuration?.["order"];
|
|
47
|
+
if (!isIntArray(order)) {
|
|
48
|
+
return { issues, context: DROPPED }; // shape errors are the schema layer's
|
|
49
|
+
}
|
|
50
|
+
const orderPath = [...path, index, "configuration", "order"];
|
|
51
|
+
let sound = true;
|
|
52
|
+
if (!isPermutation(order)) {
|
|
53
|
+
sound = false;
|
|
54
|
+
issues.push({
|
|
55
|
+
path: orderPath,
|
|
56
|
+
message: `expected a permutation of the integers 0..${order.length - 1}`,
|
|
57
|
+
kind: "invalid_value",
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
if (context.dims !== undefined && order.length !== context.dims) {
|
|
61
|
+
sound = false;
|
|
62
|
+
issues.push({
|
|
63
|
+
path: orderPath,
|
|
64
|
+
message: `expected one entry per array dimension (${context.dims})`,
|
|
65
|
+
kind: "invalid_value",
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
if (!sound) return { issues, context: DROPPED };
|
|
69
|
+
const sizes = context.chunkSizes;
|
|
70
|
+
return {
|
|
71
|
+
issues,
|
|
72
|
+
context: {
|
|
73
|
+
dims: context.dims,
|
|
74
|
+
chunkSizes:
|
|
75
|
+
sizes !== undefined && order.length === sizes.length
|
|
76
|
+
? order.map((axis) => sizes[axis] as number[])
|
|
77
|
+
: undefined,
|
|
78
|
+
},
|
|
79
|
+
};
|
|
80
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { named, simple, type DataTypeDescriptor } from "./descriptor.js";
|
|
2
|
+
|
|
3
|
+
/** The core `bool` data type. */
|
|
4
|
+
export const bool: DataTypeDescriptor = {
|
|
5
|
+
matches: named("bool"),
|
|
6
|
+
fillIssues: (fill) =>
|
|
7
|
+
typeof fill === "boolean" ? [] : simple('expected a boolean fill value for data type "bool"'),
|
|
8
|
+
};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { isByteArray, named, simple, type DataTypeDescriptor } from "./descriptor.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Fill value of the `bytes` data type: an array of integers in `[0, 255]`
|
|
5
|
+
* (one per byte) or a standard-alphabet base64 string.
|
|
6
|
+
*/
|
|
7
|
+
export type BytesFillValue = number[] | string;
|
|
8
|
+
|
|
9
|
+
const BASE64 = /^[A-Za-z0-9+/]*={0,2}$/;
|
|
10
|
+
|
|
11
|
+
/** Whether `value` is standard-alphabet base64 (padded length a multiple of 4). */
|
|
12
|
+
export function isBase64(value: string): boolean {
|
|
13
|
+
return value.length % 4 === 0 && BASE64.test(value);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** The zarr-extensions `bytes` data type (variable-length raw bytes). */
|
|
17
|
+
export const bytes: DataTypeDescriptor = {
|
|
18
|
+
matches: named("bytes"),
|
|
19
|
+
fillIssues: (fill) => {
|
|
20
|
+
if (isByteArray(fill)) return [];
|
|
21
|
+
if (typeof fill === "string" && isBase64(fill)) return [];
|
|
22
|
+
return simple(
|
|
23
|
+
'expected an array of integers in [0, 255] or a base64 string for data type "bytes"',
|
|
24
|
+
);
|
|
25
|
+
},
|
|
26
|
+
};
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The data type descriptor interface and the shared fill-value machinery
|
|
3
|
+
* the per-type modules build on. One module per supported data type;
|
|
4
|
+
* `index.ts` assembles them into the dispatch the semantic layer uses.
|
|
5
|
+
*/
|
|
6
|
+
import type { IssueKind, PathedIssue } from "../errors.js";
|
|
7
|
+
|
|
8
|
+
/** Context handed to a descriptor's fill check. */
|
|
9
|
+
export interface FillContext {
|
|
10
|
+
/** The data_type field's configuration, when present and an object. */
|
|
11
|
+
configuration: Record<string, unknown> | undefined;
|
|
12
|
+
/** Recursion depth (struct fields may nest structs). */
|
|
13
|
+
depth: number;
|
|
14
|
+
/**
|
|
15
|
+
* Recursive check for compound types: issues for `fill` judged against
|
|
16
|
+
* the data type named by `dataTypeField`, pathed relative to that fill.
|
|
17
|
+
*/
|
|
18
|
+
fillIssuesFor(dataTypeField: unknown, fill: unknown, depth: number): PathedIssue[];
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Everything the semantic layer knows about one data type. */
|
|
22
|
+
export interface DataTypeDescriptor {
|
|
23
|
+
/** Whether this module owns the given `data_type` name. */
|
|
24
|
+
matches(name: string): boolean;
|
|
25
|
+
/** Required configuration members, when the type needs a configuration. */
|
|
26
|
+
requiredConfigKeys?: readonly string[];
|
|
27
|
+
/** Name-level validity beyond ownership (e.g. r<N>'s multiple-of-8 rule). */
|
|
28
|
+
nameIssue?(name: string): string | undefined;
|
|
29
|
+
/** Fill-value issues, pathed relative to the fill value itself. */
|
|
30
|
+
fillIssues(fill: unknown, name: string, context: FillContext): PathedIssue[];
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function issue(
|
|
34
|
+
path: ReadonlyArray<string | number>,
|
|
35
|
+
message: string,
|
|
36
|
+
kind: IssueKind = "invalid_value",
|
|
37
|
+
): PathedIssue {
|
|
38
|
+
return { path, message, kind };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** A single unpathed issue — the common case for scalar types. */
|
|
42
|
+
export function simple(message: string): PathedIssue[] {
|
|
43
|
+
return [issue([], message)];
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function named(matchName: string): (name: string) => boolean {
|
|
47
|
+
return (name) => name === matchName;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// --- shared fill forms ------------------------------------------------------
|
|
51
|
+
|
|
52
|
+
/** A float fill value: a JSON number, a non-finite sentinel, or a hex string. */
|
|
53
|
+
export type FloatFillValue = number | "NaN" | "Infinity" | "-Infinity" | string;
|
|
54
|
+
|
|
55
|
+
/** A complex fill value: `[real, imaginary]`, each a float fill form. */
|
|
56
|
+
export type ComplexFillValue = [FloatFillValue, FloatFillValue];
|
|
57
|
+
|
|
58
|
+
export function isFloatFill(value: unknown, hexDigits: number): boolean {
|
|
59
|
+
if (typeof value === "number") return true;
|
|
60
|
+
if (typeof value !== "string") return false;
|
|
61
|
+
if (value === "NaN" || value === "Infinity" || value === "-Infinity") return true;
|
|
62
|
+
return new RegExp(`^0x[0-9a-fA-F]{${hexDigits}}$`).test(value);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function floatFillMessage(name: string, hexDigits: number): string {
|
|
66
|
+
return (
|
|
67
|
+
`expected a number, "NaN", "Infinity", "-Infinity", or a ` +
|
|
68
|
+
`${hexDigits}-hex-digit "0x..." string for data type ${JSON.stringify(name)}`
|
|
69
|
+
);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function isByteArray(value: unknown): value is number[] {
|
|
73
|
+
return (
|
|
74
|
+
Array.isArray(value) &&
|
|
75
|
+
Object.keys(value).length === value.length &&
|
|
76
|
+
value.every((item) => Number.isInteger(item) && (item as number) >= 0 && (item as number) <= 255)
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// --- per-family factories ---------------------------------------------------
|
|
81
|
+
|
|
82
|
+
export function intDataType(name: string, low: number, high: number): DataTypeDescriptor {
|
|
83
|
+
return {
|
|
84
|
+
matches: named(name),
|
|
85
|
+
fillIssues: (fill) =>
|
|
86
|
+
Number.isInteger(fill) && (fill as number) >= low && (fill as number) <= high
|
|
87
|
+
? []
|
|
88
|
+
: simple(`expected an integer in [${low}, ${high}] for data type ${JSON.stringify(name)}`),
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function floatDataType(name: string, hexDigits: number): DataTypeDescriptor {
|
|
93
|
+
return {
|
|
94
|
+
matches: named(name),
|
|
95
|
+
fillIssues: (fill) =>
|
|
96
|
+
isFloatFill(fill, hexDigits) ? [] : simple(floatFillMessage(name, hexDigits)),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function complexDataType(name: string, componentHexDigits: number): DataTypeDescriptor {
|
|
101
|
+
return {
|
|
102
|
+
matches: named(name),
|
|
103
|
+
fillIssues: (fill) => {
|
|
104
|
+
const ok =
|
|
105
|
+
Array.isArray(fill) &&
|
|
106
|
+
fill.length === 2 &&
|
|
107
|
+
fill.every((part) => isFloatFill(part, componentHexDigits));
|
|
108
|
+
return ok
|
|
109
|
+
? []
|
|
110
|
+
: simple(
|
|
111
|
+
`expected a two-element [real, imaginary] array for data type ${JSON.stringify(name)}`,
|
|
112
|
+
);
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The data types this package interprets, one module per type, assembled
|
|
3
|
+
* into the dispatch the semantic layer uses: the core scalars, the r<N>
|
|
4
|
+
* raw-bits family, and the established zarr-extensions conventions
|
|
5
|
+
* (string, bytes, numpy.datetime64/timedelta64, struct). Unrecognized
|
|
6
|
+
* names yield no verdicts — the name space is open — and other types'
|
|
7
|
+
* required fields remain registry-schema facts.
|
|
8
|
+
*/
|
|
9
|
+
import type { PathedIssue } from "../errors.js";
|
|
10
|
+
import { configurationMissing, fieldParts } from "../guards.js";
|
|
11
|
+
import { bool } from "./bool.js";
|
|
12
|
+
import { bytes } from "./bytes.js";
|
|
13
|
+
import { complex64 } from "./complex64.js";
|
|
14
|
+
import { complex128 } from "./complex128.js";
|
|
15
|
+
import { issue, type DataTypeDescriptor, type FillContext } from "./descriptor.js";
|
|
16
|
+
import { float16 } from "./float16.js";
|
|
17
|
+
import { float32 } from "./float32.js";
|
|
18
|
+
import { float64 } from "./float64.js";
|
|
19
|
+
import { int8 } from "./int8.js";
|
|
20
|
+
import { int16 } from "./int16.js";
|
|
21
|
+
import { int32 } from "./int32.js";
|
|
22
|
+
import { int64 } from "./int64.js";
|
|
23
|
+
import { numpyDatetime64 } from "./numpy-datetime64.js";
|
|
24
|
+
import { numpyTimedelta64 } from "./numpy-timedelta64.js";
|
|
25
|
+
import { raw } from "./raw.js";
|
|
26
|
+
import { string } from "./string.js";
|
|
27
|
+
import { struct } from "./struct.js";
|
|
28
|
+
import { uint8 } from "./uint8.js";
|
|
29
|
+
import { uint16 } from "./uint16.js";
|
|
30
|
+
import { uint32 } from "./uint32.js";
|
|
31
|
+
import { uint64 } from "./uint64.js";
|
|
32
|
+
|
|
33
|
+
export type {
|
|
34
|
+
ComplexFillValue,
|
|
35
|
+
DataTypeDescriptor,
|
|
36
|
+
FillContext,
|
|
37
|
+
FloatFillValue,
|
|
38
|
+
} from "./descriptor.js";
|
|
39
|
+
export type { BytesFillValue } from "./bytes.js";
|
|
40
|
+
export type {
|
|
41
|
+
NumpyDatetime64Configuration,
|
|
42
|
+
NumpyDatetime64FillValue,
|
|
43
|
+
NumpyTimeUnit,
|
|
44
|
+
} from "./numpy-datetime64.js";
|
|
45
|
+
export type { RawBytesFillValue } from "./raw.js";
|
|
46
|
+
export type { StringFillValue } from "./string.js";
|
|
47
|
+
export type { StructConfiguration, StructField } from "./struct.js";
|
|
48
|
+
|
|
49
|
+
const DESCRIPTORS: readonly DataTypeDescriptor[] = [
|
|
50
|
+
bool,
|
|
51
|
+
int8, int16, int32, int64,
|
|
52
|
+
uint8, uint16, uint32, uint64,
|
|
53
|
+
float16, float32, float64,
|
|
54
|
+
complex64, complex128,
|
|
55
|
+
raw,
|
|
56
|
+
string, bytes,
|
|
57
|
+
numpyDatetime64, numpyTimedelta64,
|
|
58
|
+
struct,
|
|
59
|
+
];
|
|
60
|
+
|
|
61
|
+
function descriptorFor(name: string): DataTypeDescriptor | undefined {
|
|
62
|
+
return DESCRIPTORS.find((descriptor) => descriptor.matches(name));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// struct fields may nest structs; matches the structural layer's cap.
|
|
66
|
+
const MAX_FILL_DEPTH = 64;
|
|
67
|
+
|
|
68
|
+
function fillIssuesFor(dataTypeField: unknown, fill: unknown, depth: number): PathedIssue[] {
|
|
69
|
+
if (depth >= MAX_FILL_DEPTH) return [];
|
|
70
|
+
const parts = fieldParts(dataTypeField);
|
|
71
|
+
if (parts === undefined) return [];
|
|
72
|
+
const descriptor = descriptorFor(parts.name);
|
|
73
|
+
if (descriptor === undefined) return []; // unrecognized: no verdict
|
|
74
|
+
const context: FillContext = { configuration: parts.configuration, depth, fillIssuesFor };
|
|
75
|
+
return descriptor.fillIssues(fill, parts.name, context);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Interpret a document's `data_type` field against its `fill_value`:
|
|
80
|
+
* name-level rules, required configuration members, and the fill's JSON
|
|
81
|
+
* shape (recursively, for struct). Issues are pathed from the document
|
|
82
|
+
* root (`data_type` / `fill_value`).
|
|
83
|
+
*/
|
|
84
|
+
export function dataTypeVerdict(
|
|
85
|
+
rawField: unknown,
|
|
86
|
+
fillPresent: boolean,
|
|
87
|
+
fill: unknown,
|
|
88
|
+
): PathedIssue[] {
|
|
89
|
+
const parts = fieldParts(rawField);
|
|
90
|
+
if (parts === undefined) return [];
|
|
91
|
+
const descriptor = descriptorFor(parts.name);
|
|
92
|
+
if (descriptor === undefined) return [];
|
|
93
|
+
const issues: PathedIssue[] = [];
|
|
94
|
+
const nameProblem = descriptor.nameIssue?.(parts.name);
|
|
95
|
+
if (nameProblem !== undefined) {
|
|
96
|
+
issues.push(issue(["data_type"], nameProblem));
|
|
97
|
+
}
|
|
98
|
+
const required = descriptor.requiredConfigKeys;
|
|
99
|
+
if (required !== undefined) {
|
|
100
|
+
if (configurationMissing(rawField)) {
|
|
101
|
+
issues.push(
|
|
102
|
+
issue(
|
|
103
|
+
["data_type"],
|
|
104
|
+
`${JSON.stringify(parts.name)} requires a configuration with ${required
|
|
105
|
+
.map((key) => JSON.stringify(key))
|
|
106
|
+
.join(" and ")}`,
|
|
107
|
+
"missing_key",
|
|
108
|
+
),
|
|
109
|
+
);
|
|
110
|
+
} else if (parts.configuration !== undefined) {
|
|
111
|
+
for (const key of required) {
|
|
112
|
+
if (!Object.hasOwn(parts.configuration, key)) {
|
|
113
|
+
issues.push(
|
|
114
|
+
issue(["data_type", "configuration", key], "missing required key", "missing_key"),
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (fillPresent) {
|
|
121
|
+
issues.push(
|
|
122
|
+
...fillIssuesFor(rawField, fill, 0).map((inner) => ({
|
|
123
|
+
...inner,
|
|
124
|
+
path: ["fill_value", ...inner.path],
|
|
125
|
+
})),
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
return issues;
|
|
129
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { named, simple, type DataTypeDescriptor } from "./descriptor.js";
|
|
2
|
+
|
|
3
|
+
/** Time unit codes used by numpy.datetime64 / numpy.timedelta64. */
|
|
4
|
+
export type NumpyTimeUnit =
|
|
5
|
+
| "Y" | "M" | "W" | "D" | "h" | "m" | "s"
|
|
6
|
+
| "ms" | "us" | "μs" | "ns" | "ps" | "fs" | "as" | "generic";
|
|
7
|
+
|
|
8
|
+
/** Configuration of the numpy temporal data types. */
|
|
9
|
+
export interface NumpyDatetime64Configuration {
|
|
10
|
+
unit: NumpyTimeUnit;
|
|
11
|
+
scale_factor: number;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Fill value of the numpy temporal data types: an integer count of
|
|
16
|
+
* `unit * scale_factor` since the epoch, or the `"NaT"` sentinel.
|
|
17
|
+
*/
|
|
18
|
+
export type NumpyDatetime64FillValue = number | "NaT";
|
|
19
|
+
|
|
20
|
+
/** Descriptor factory shared by numpy.datetime64 and numpy.timedelta64. */
|
|
21
|
+
export function numpyTemporalDataType(name: string): DataTypeDescriptor {
|
|
22
|
+
return {
|
|
23
|
+
matches: named(name),
|
|
24
|
+
requiredConfigKeys: ["unit", "scale_factor"],
|
|
25
|
+
fillIssues: (fill) =>
|
|
26
|
+
Number.isInteger(fill) || fill === "NaT"
|
|
27
|
+
? []
|
|
28
|
+
: simple(`expected an integer or "NaT" for data type ${JSON.stringify(name)}`),
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** The zarr-extensions `numpy.datetime64` data type. */
|
|
33
|
+
export const numpyDatetime64 = numpyTemporalDataType("numpy.datetime64");
|