zarr-metadata 0.7.0 → 0.8.0

package/src/chunk-grids.ts

@@ -0,0 +1,158 @@
+/**
+ * The chunk grids this package interprets: syntax (configuration types) and
+ * semantics (the spec's prose rules) for `regular` and `rectilinear`, in
+ * one place per grid. The structural document layer never reads this
+ * module — extension points are uninterpreted there by spec design.
+ */
+
+import type { PathedIssue } from "./errors.js";
+import { configurationMissing, fieldParts, isIntArray, isPlainObject } from "./guards.js";
+
+/** Configuration of the core `regular` chunk grid. */
+export interface RegularChunkGridConfiguration {
+  chunk_shape: number[];
+}
+
+/**
+ * One dimension's spec in a rectilinear grid: a bare integer (uniform
+ * shorthand, no sum constraint) or a list of chunk sizes and
+ * `[size, count]` run-length pairs that must sum to the dimension length.
+ */
+export type RectilinearDimSpec = number | Array<number | [number, number]>;
+
+/** Configuration of the zarr-extensions `rectilinear` chunk grid. */
+export interface RectilinearChunkGridConfiguration {
+  kind: "inline";
+  chunk_shapes: RectilinearDimSpec[];
+}
+
+/** What interpreting a chunk grid yields for the rest of the semantic layer. */
+export interface ChunkGridVerdict {
+  /** Issues, pathed relative to the `chunk_grid` field. */
+  issues: PathedIssue[];
+  /**
+   * Distinct chunk lengths per dimension — the division context a
+   * sharding codec's inner chunks must satisfy — when derivable.
+   */
+  chunkSizes: number[][] | undefined;
+}
+
+function regularIssues(rawGrid: unknown, shape: number[] | undefined): ChunkGridVerdict {
+  const issues: PathedIssue[] = [];
+  let chunkSizes: number[][] | undefined;
+  const configuration = fieldParts(rawGrid)?.configuration;
+  if (configurationMissing(rawGrid)) {
+    issues.push({
+      path: [],
+      message: '"regular" requires a configuration with "chunk_shape"',
+      kind: "missing_key",
+    });
+  } else if (configuration !== undefined && !Object.hasOwn(configuration, "chunk_shape")) {
+    issues.push({
+      path: ["configuration", "chunk_shape"],
+      message: "missing required key",
+      kind: "missing_key",
+    });
+  }
+  const configured = configuration?.["chunk_shape"];
+  if (isIntArray(configured)) {
+    if (shape !== undefined && configured.length !== shape.length) {
+      issues.push({
+        path: ["configuration", "chunk_shape"],
+        message: `expected one length per dimension of shape (${shape.length})`,
+        kind: "invalid_value",
+      });
+      // wrong arity: unusable as division context
+    } else {
+      chunkSizes = configured.map((length) => [length]);
+    }
+  }
+  return { issues, chunkSizes };
+}
+
+function rectilinearIssues(rawGrid: unknown, shape: number[] | undefined): ChunkGridVerdict {
+  const issues: PathedIssue[] = [];
+  let chunkSizes: number[][] | undefined;
+  const configuration = fieldParts(rawGrid)?.configuration;
+  if (configurationMissing(rawGrid)) {
+    issues.push({
+      path: [],
+      message: '"rectilinear" requires a configuration with "kind" and "chunk_shapes"',
+      kind: "missing_key",
+    });
+  } else if (configuration !== undefined) {
+    for (const key of ["kind", "chunk_shapes"]) {
+      if (!Object.hasOwn(configuration, key)) {
+        issues.push({
+          path: ["configuration", key],
+          message: "missing required key",
+          kind: "missing_key",
+        });
+      }
+    }
+  }
+  const specs = configuration?.["chunk_shapes"];
+  if (Array.isArray(specs)) {
+    if (shape !== undefined && specs.length !== shape.length) {
+      issues.push({
+        path: ["configuration", "chunk_shapes"],
+        message: `expected one entry per dimension of shape (${shape.length})`,
+        kind: "invalid_value",
+      });
+    } else {
+      const perDim: (number[] | undefined)[] = specs.map((spec, dim) => {
+        // Bare integer: uniform shorthand, no sum constraint (edge chunks
+        // are permitted, as with the regular grid).
+        if (Number.isInteger(spec)) return (spec as number) > 0 ? [spec as number] : undefined;
+        if (!Array.isArray(spec)) return undefined; // registry schema's problem
+        const sizes = new Set<number>();
+        let total = 0;
+        for (const entry of spec) {
+          if (Number.isInteger(entry)) {
+            sizes.add(entry as number);
+            total += entry as number;
+          } else if (
+            Array.isArray(entry) &&
+            entry.length === 2 &&
+            Number.isInteger(entry[0]) &&
+            Number.isInteger(entry[1])
+          ) {
+            sizes.add(entry[0] as number);
+            total += (entry[0] as number) * (entry[1] as number);
+          } else {
+            return undefined; // malformed entry: registry schema's problem
+          }
+        }
+        const extent = shape?.[dim];
+        if (extent !== undefined && total !== extent) {
+          issues.push({
+            path: ["configuration", "chunk_shapes", dim],
+            message: `expected chunk sizes summing to ${extent} along dimension ${dim}, got ${total}`,
+            kind: "invalid_value",
+          });
+        }
+        return [...sizes];
+      });
+      if (perDim.every((sizes) => sizes !== undefined)) {
+        chunkSizes = perDim as number[][];
+      }
+    }
+  }
+  return { issues, chunkSizes };
+}
+
+/**
+ * Interpret a document's `chunk_grid` field: required members, cross-field
+ * rules against `shape`, and the division context for codec pipelines.
+ * Unrecognized grid names yield no issues and no context — the extension
+ * name space is open.
+ */
+export function chunkGridVerdict(rawGrid: unknown, shape: number[] | undefined): ChunkGridVerdict {
+  if (!isPlainObject(rawGrid) && typeof rawGrid !== "string") {
+    return { issues: [], chunkSizes: undefined }; // structural layer's problem
+  }
+  const name = fieldParts(rawGrid)?.name;
+  if (name === "regular") return regularIssues(rawGrid, shape);
+  if (name === "rectilinear") return rectilinearIssues(rawGrid, shape);
+  return { issues: [], chunkSizes: undefined };
+}

package/src/codecs.ts

@@ -0,0 +1,214 @@
+/**
+ * The codecs this package interprets — `transpose` and `sharding_indexed` —
+ * syntax (configuration types) and semantics together, plus the pipeline
+ * walker that threads dimensional context between codecs (mirroring
+ * zarr-python's chunk-spec threading): a valid transpose permutes the
+ * per-dimension chunk sizes for the codecs after it, and any codec this
+ * module cannot reason about — `reshape` may change a chunk's rank, and
+ * unknown codecs may do anything — invalidates the dimensional context for
+ * the rest of the pipeline instead of letting stale array-level facts
+ * produce false verdicts.
+ *
+ * Other codecs' required fields (blosc, gzip, zstd, ...) are deliberately
+ * NOT known here: they are registry-schema facts enforced by schema-driven
+ * tooling, and a second hardcoded source of truth would drift.
+ */
+
+import type { PathedIssue } from "./errors.js";
+import { configurationMissing, fieldParts, isIntArray, type Path } from "./guards.js";
+
+/** Configuration of the core `transpose` codec. */
+export interface TransposeCodecConfiguration {
+  order: number[];
+}
+
+/** Configuration of the core `sharding_indexed` codec. */
+export interface ShardingIndexedCodecConfiguration {
+  chunk_shape: number[];
+  codecs: unknown[];
+  index_codecs: unknown[];
+  index_location?: "start" | "end";
+}
+
+function isPermutation(order: number[]): boolean {
+  const seen = new Set(order);
+  return seen.size === order.length && order.every((entry) => entry >= 0 && entry < order.length);
+}
+
+/**
+ * Walk one codec pipeline. `initialDims` is the array dimensionality at
+ * this point (undefined when unknowable); `initialChunkSizes` holds the
+ * distinct chunk lengths per dimension this pipeline may encode — what a
+ * sharding codec's inner chunks must divide. A regular grid contributes
+ * one size per dimension; a rectilinear grid may contribute several.
+ */
+export function pipelineIssues(
+  pipeline: unknown[],
+  path: Path,
+  initialDims: number | undefined,
+  initialChunkSizes: number[][] | undefined,
+): PathedIssue[] {
+  const issues: PathedIssue[] = [];
+  // The dimensional context THREADS through the pipeline: transpose permutes
+  // it, and anything this module cannot reason about invalidates it for the
+  // remaining codecs.
+  let dims = initialDims;
+  let chunkSizes = initialChunkSizes;
+  const dropContext = (): void => {
+    dims = undefined;
+    chunkSizes = undefined;
+  };
+  pipeline.forEach((entry, index) => {
+    const parts = fieldParts(entry);
+    if (parts === undefined) {
+      dropContext(); // structurally invalid entry: no further reasoning
+      return;
+    }
+    const { name, configuration } = parts;
+    const configMissing = configurationMissing(entry);
+    if (name === "transpose") {
+      if (configMissing) {
+        issues.push({
+          path: [...path, index],
+          message: '"transpose" requires a configuration with "order"',
+          kind: "missing_key",
+        });
+        dropContext();
+        return;
+      }
+      if (configuration !== undefined && !Object.hasOwn(configuration, "order")) {
+        issues.push({
+          path: [...path, index, "configuration", "order"],
+          message: "missing required key",
+          kind: "missing_key",
+        });
+        dropContext();
+        return;
+      }
+      const order = configuration?.["order"];
+      if (!isIntArray(order)) {
+        dropContext(); // shape errors are the schema layer's
+        return;
+      }
+      const orderPath = [...path, index, "configuration", "order"];
+      let sound = true;
+      if (!isPermutation(order)) {
+        sound = false;
+        issues.push({
+          path: orderPath,
+          message: `expected a permutation of the integers 0..${order.length - 1}`,
+          kind: "invalid_value",
+        });
+      }
+      if (dims !== undefined && order.length !== dims) {
+        sound = false;
+        issues.push({
+          path: orderPath,
+          message: `expected one entry per array dimension (${dims})`,
+          kind: "invalid_value",
+        });
+      }
+      if (!sound) {
+        dropContext();
+      } else if (chunkSizes !== undefined) {
+        const sizes = chunkSizes;
+        chunkSizes =
+          order.length === sizes.length
+            ? order.map((axis) => sizes[axis] as number[])
+            : undefined;
+      }
+    } else if (name === "sharding_indexed") {
+      if (configMissing) {
+        issues.push({
+          path: [...path, index],
+          message:
+            '"sharding_indexed" requires a configuration with "chunk_shape", "codecs", and "index_codecs"',
+          kind: "missing_key",
+        });
+        dropContext();
+        return;
+      }
+      if (configuration !== undefined) {
+        for (const key of ["chunk_shape", "codecs", "index_codecs"]) {
+          if (!Object.hasOwn(configuration, key)) {
+            issues.push({
+              path: [...path, index, "configuration", key],
+              message: "missing required key",
+              kind: "missing_key",
+            });
+          }
+        }
+      }
+      const chunkShape = configuration?.["chunk_shape"];
+      if (!isIntArray(chunkShape)) {
+        // Still walk a present inner pipeline (entries deserve their own
+        // verdicts) before the context becomes unknowable.
+        const inner = configuration?.["codecs"];
+        if (Array.isArray(inner)) {
+          issues.push(
+            ...pipelineIssues(
+              inner,
+              [...path, index, "configuration", "codecs"],
+              undefined,
+              undefined,
+            ),
+          );
+        }
+        dropContext();
+        return;
+      }
+      const chunkShapePath = [...path, index, "configuration", "chunk_shape"];
+      if (dims !== undefined && chunkShape.length !== dims) {
+        issues.push({
+          path: chunkShapePath,
+          message: `expected one length per array dimension (${dims})`,
+          kind: "invalid_value",
+        });
+      } else if (
+        chunkSizes !== undefined &&
+        chunkShape.length === chunkSizes.length &&
+        chunkShape.every((length) => length > 0)
+      ) {
+        let violation: { axis: number; size: number } | undefined;
+        chunkSizes.forEach((sizes, axis) => {
+          if (violation !== undefined) return;
+          const bad = sizes.find((size) => size % (chunkShape[axis] as number) !== 0);
+          if (bad !== undefined) violation = { axis, size: bad };
+        });
+        if (violation !== undefined) {
+          const { axis, size } = violation;
+          const sizesNow = chunkSizes;
+          const uniform = sizesNow.every((sizes) => sizes.length === 1);
+          issues.push({
+            path: chunkShapePath,
+            message: uniform
+              ? `expected ${JSON.stringify(chunkShape)} to evenly divide the outer chunk shape ${JSON.stringify(sizesNow.map((sizes) => sizes[0]))}`
+              : `expected ${JSON.stringify(chunkShape)} to evenly divide every chunk size of the grid (dimension ${axis} has chunk size ${size})`,
+            kind: "invalid_value",
+          });
+        }
+      }
+      const inner = configuration?.["codecs"];
+      if (Array.isArray(inner)) {
+        issues.push(
+          ...pipelineIssues(
+            inner,
+            [...path, index, "configuration", "codecs"],
+            chunkShape.length,
+            chunkShape.map((length) => [length]),
+          ),
+        );
+      }
+      // index_codecs encode the shard index, whose shape differs from the
+      // array's — no dimensional context applies there. After the
+      // array -> bytes boundary the dimensional context is spent.
+      dropContext();
+    } else {
+      // A codec this module cannot reason about (reshape may change a
+      // chunk's rank; unknown codecs may do anything): stale array-level
+      // facts must not judge the codecs after it.
+      dropContext();
+    }
+  });
+  return issues;
+}

package/src/data-types.ts

@@ -0,0 +1,81 @@
+/**
+ * The core data types this package interprets: fill-value syntax per type,
+ * in one place per family. Extension data types are skipped — their shapes
+ * are registry-schema facts, and the name space is open.
+ */
+
+/** A float fill value: a JSON number, a non-finite sentinel, or a hex string. */
+export type FloatFillValue = number | "NaN" | "Infinity" | "-Infinity" | string;
+
+/** A complex fill value: `[real, imaginary]`, each a float fill form. */
+export type ComplexFillValue = [FloatFillValue, FloatFillValue];
+
+const INT_RANGES: Record<string, readonly [number, number]> = {
+  int8: [-128, 127],
+  int16: [-32768, 32767],
+  int32: [-2147483648, 2147483647],
+  int64: [-9223372036854775808, 9223372036854775807],
+  uint8: [0, 255],
+  uint16: [0, 65535],
+  uint32: [0, 4294967295],
+  uint64: [0, 18446744073709551615],
+};
+
+const FLOAT_HEX_DIGITS: Record<string, number> = {
+  float16: 4,
+  float32: 8,
+  float64: 16,
+};
+
+/** Hex digit width of each complex type's component floats. */
+const COMPLEX_COMPONENT_HEX_DIGITS: Record<string, number> = {
+  complex64: 8,
+  complex128: 16,
+};
+
+function isFloatFill(value: unknown, hexDigits: number): boolean {
+  if (typeof value === "number") return true;
+  if (typeof value !== "string") return false;
+  if (value === "NaN" || value === "Infinity" || value === "-Infinity") return true;
+  return new RegExp(`^0x[0-9a-fA-F]{${hexDigits}}$`).test(value);
+}
+
+function floatFillMessage(name: string, hexDigits: number): string {
+  return (
+    `expected a number, "NaN", "Infinity", "-Infinity", or a ` +
+    `${hexDigits}-hex-digit "0x..." string for data type ${JSON.stringify(name)}`
+  );
+}
+
+/** The fill_value issue for a KNOWN core data type, if any; undefined otherwise. */
+export function fillValueIssue(dataTypeName: string, fill: unknown): string | undefined {
+  if (dataTypeName === "bool") {
+    return typeof fill === "boolean"
+      ? undefined
+      : 'expected a boolean fill value for data type "bool"';
+  }
+  const intRange = INT_RANGES[dataTypeName];
+  if (intRange !== undefined) {
+    const [low, high] = intRange;
+    return Number.isInteger(fill) && (fill as number) >= low && (fill as number) <= high
+      ? undefined
+      : `expected an integer in [${low}, ${high}] for data type ${JSON.stringify(dataTypeName)}`;
+  }
+  const floatDigits = FLOAT_HEX_DIGITS[dataTypeName];
+  if (floatDigits !== undefined) {
+    return isFloatFill(fill, floatDigits)
+      ? undefined
+      : floatFillMessage(dataTypeName, floatDigits);
+  }
+  const complexDigits = COMPLEX_COMPONENT_HEX_DIGITS[dataTypeName];
+  if (complexDigits !== undefined) {
+    const ok =
+      Array.isArray(fill) &&
+      fill.length === 2 &&
+      fill.every((part) => isFloatFill(part, complexDigits));
+    return ok
+      ? undefined
+      : `expected a two-element [real, imaginary] array for data type ${JSON.stringify(dataTypeName)}`;
+  }
+  return undefined; // unrecognized data type: no verdict
+}

package/src/guards.ts

@@ -0,0 +1,48 @@
+/**
+ * Shared value guards for the content modules (chunk grids, codecs, data
+ * types) and the semantics orchestrator. The structural document layer
+ * (validation.ts) keeps its own copies deliberately: it is the
+ * corpus-governed layer and stays self-contained.
+ */
+
+export type Path = ReadonlyArray<string | number>;
+
+export function isPlainObject(value: unknown): value is Record<string, unknown> {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
+  const proto: unknown = Object.getPrototypeOf(value);
+  return proto === null || proto === Object.prototype;
+}
+
+export function isIntArray(value: unknown): value is number[] {
+  return (
+    Array.isArray(value) &&
+    Object.keys(value).length === value.length &&
+    value.every((item) => Number.isInteger(item))
+  );
+}
+
+/** The `{name, configuration}` of a metadata field, when extractable. */
+export function fieldParts(
+  field: unknown,
+): { name: string; configuration: Record<string, unknown> | undefined } | undefined {
+  if (typeof field === "string") return { name: field, configuration: undefined };
+  if (isPlainObject(field) && typeof field["name"] === "string") {
+    const configuration = field["configuration"];
+    return {
+      name: field["name"],
+      configuration: isPlainObject(configuration) ? configuration : undefined,
+    };
+  }
+  return undefined;
+}
+
+/**
+ * Whether a metadata field genuinely lacks a `configuration` member (a
+ * malformed one — `configuration: 5` — is the structural layer's
+ * complaint, not the semantic layer's).
+ */
+export function configurationMissing(field: unknown): boolean {
+  return (
+    typeof field === "string" || (isPlainObject(field) && !Object.hasOwn(field, "configuration"))
+  );
+}

package/src/index.ts

@@ -20,6 +20,17 @@ export {
   type ParseResult,
   type PathedIssue,
 } from "./errors.js";
+export type {
+  ChunkGridVerdict,
+  RectilinearChunkGridConfiguration,
+  RectilinearDimSpec,
+  RegularChunkGridConfiguration,
+} from "./chunk-grids.js";
+export type {
+  ShardingIndexedCodecConfiguration,
+  TransposeCodecConfiguration,
+} from "./codecs.js";
+export type { ComplexFillValue, FloatFillValue } from "./data-types.js";
 export * from "./schemas.js";
 export * from "./semantics.js";
 export type { StandardSchemaV1 } from "./standard-schema.js";