bio2zarr 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

bio2zarr/{vcf2zarr/vcz.py → vcz.py}

@@ -1,41 +1,29 @@
-import contextlib
+import abc
 import dataclasses
 import json
 import logging
 import os
-import os.path
 import pathlib
 import shutil
-import tempfile
 
-import humanfriendly
 import numcodecs
 import numpy as np
 import zarr
 
-from bio2zarr.zarr_utils import ZARR_FORMAT_KWARGS, zarr_v3
-
-from .. import constants, core, provenance
-from . import icf
+from bio2zarr import constants, core, provenance, zarr_utils
 
 logger = logging.getLogger(__name__)
 
-
-def inspect(path):
-    path = pathlib.Path(path)
-    if not path.exists():
-        raise ValueError(f"Path not found: {path}")
-    if (path / "metadata.json").exists():
-        obj = icf.IntermediateColumnarFormat(path)
-    # NOTE: this is too strict, we should support more general Zarrs, see #276
-    elif (path / ".zmetadata").exists():
-        obj = VcfZarr(path)
-    else:
-        raise ValueError(f"{path} not in ICF or VCF Zarr format")
-    return obj.summary_table()
-
-
+ZARR_SCHEMA_FORMAT_VERSION = "0.6"
+DEFAULT_VARIANT_CHUNK_SIZE = 1000
+DEFAULT_SAMPLE_CHUNK_SIZE = 10_000
 DEFAULT_ZARR_COMPRESSOR = numcodecs.Blosc(cname="zstd", clevel=7)
+DEFAULT_ZARR_COMPRESSOR_GENOTYPES = numcodecs.Blosc(
+    cname="zstd", clevel=7, shuffle=numcodecs.Blosc.BITSHUFFLE
+)
+DEFAULT_ZARR_COMPRESSOR_BOOL = numcodecs.Blosc(
+    cname="zstd", clevel=7, shuffle=numcodecs.Blosc.BITSHUFFLE
+)
 
 _fixed_field_descriptions = {
     "variant_contig": "An identifier from the reference genome or an angle-bracketed ID"
@@ -49,131 +37,254 @@ _fixed_field_descriptions = {
 }
 
 
+@dataclasses.dataclass
+class VariantData:
+    """Represents variant data returned by iter_alleles_and_genotypes."""
+
+    variant_length: int
+    alleles: np.ndarray
+    genotypes: np.ndarray
+    phased: np.ndarray
+
+
+class Source(abc.ABC):
+    @property
+    @abc.abstractmethod
+    def path(self):
+        pass
+
+    @property
+    @abc.abstractmethod
+    def num_records(self):
+        pass
+
+    @property
+    @abc.abstractmethod
+    def num_samples(self):
+        pass
+
+    @property
+    @abc.abstractmethod
+    def samples(self):
+        pass
+
+    @property
+    def contigs(self):
+        return None
+
+    @property
+    def filters(self):
+        return None
+
+    @property
+    def root_attrs(self):
+        return {}
+
+    @abc.abstractmethod
+    def iter_alleles_and_genotypes(self, start, stop, shape, num_alleles):
+        pass
+
+    def iter_id(self, start, stop):
+        return
+
+    def iter_contig(self, start, stop):
+        return
+
+    @abc.abstractmethod
+    def iter_field(self, field_name, shape, start, stop):
+        pass
+
+    @abc.abstractmethod
+    def generate_schema(self, variants_chunk_size, samples_chunk_size, local_alleles):
+        pass
+
+
+@dataclasses.dataclass
+class VcfZarrDimension:
+    size: int
+    chunk_size: int
+
+    def asdict(self):
+        return dataclasses.asdict(self)
+
+    @classmethod
+    def fromdict(cls, d):
+        return cls(**d)
+
+    @classmethod
+    def unchunked(cls, size):
+        return cls(size, max(size, 1))
+
+
+def standard_dimensions(
+    *,
+    variants_size,
+    samples_size,
+    variants_chunk_size=None,
+    samples_chunk_size=None,
+    alleles_size=None,
+    filters_size=None,
+    ploidy_size=None,
+    genotypes_size=None,
+):
+    """
+    Returns a dictionary mapping dimension names to definition for the standard
+    fields in a VCF.
+    """
+    if variants_chunk_size is None:
+        variants_chunk_size = max(1, min(variants_size, DEFAULT_VARIANT_CHUNK_SIZE))
+    if samples_chunk_size is None:
+        samples_chunk_size = max(1, min(samples_size, DEFAULT_SAMPLE_CHUNK_SIZE))
+
+    dimensions = {
+        "variants": VcfZarrDimension(variants_size, variants_chunk_size),
+        "samples": VcfZarrDimension(samples_size, samples_chunk_size),
+    }
+
+    if alleles_size is not None:
+        dimensions["alleles"] = VcfZarrDimension.unchunked(alleles_size)
+        if alleles_size > 1:
+            dimensions["alt_alleles"] = VcfZarrDimension.unchunked(alleles_size - 1)
+
+    if filters_size is not None:
+        dimensions["filters"] = VcfZarrDimension.unchunked(filters_size)
+
+    if ploidy_size is not None:
+        dimensions["ploidy"] = VcfZarrDimension.unchunked(ploidy_size)
+
+    if genotypes_size is not None:
+        dimensions["genotypes"] = VcfZarrDimension.unchunked(genotypes_size)
+
+    return dimensions
+
+
 @dataclasses.dataclass
 class ZarrArraySpec:
     name: str
     dtype: str
-    shape: tuple
-    chunks: tuple
     dimensions: tuple
     description: str
-    vcf_field: str
-    compressor: dict
-    filters: list
+    compressor: dict = None
+    filters: list = None
+    source: str = None
 
     def __post_init__(self):
         if self.name in _fixed_field_descriptions:
             self.description = self.description or _fixed_field_descriptions[self.name]
 
-        # Ensure these are tuples for ease of comparison and consistency
-        self.shape = tuple(self.shape)
-        self.chunks = tuple(self.chunks)
         self.dimensions = tuple(self.dimensions)
-        self.filters = tuple(self.filters)
+        self.filters = tuple(self.filters) if self.filters is not None else None
 
-    @staticmethod
-    def new(**kwargs):
-        spec = ZarrArraySpec(
-            **kwargs, compressor=DEFAULT_ZARR_COMPRESSOR.get_config(), filters=[]
-        )
-        spec._choose_compressor_settings()
-        return spec
+    def get_shape(self, schema):
+        return schema.get_shape(self.dimensions)
+
+    def get_chunks(self, schema):
+        return schema.get_chunks(self.dimensions)
+
+    def get_chunk_nbytes(self, schema):
+        element_size = np.dtype(self.dtype).itemsize
+        chunks = self.get_chunks(schema)
+        shape = self.get_shape(schema)
+
+        # Calculate actual chunk size accounting for dimension limits
+        items = 1
+        for i, chunk_size in enumerate(chunks):
+            items *= min(chunk_size, shape[i])
+
+        # Include sizes for extra dimensions (if any)
+        if len(shape) > len(chunks):
+            for size in shape[len(chunks) :]:
+                items *= size
+
+        return element_size * items
 
     @staticmethod
     def from_field(
         vcf_field,
+        schema,
         *,
-        num_variants,
-        num_samples,
-        variants_chunk_size,
-        samples_chunk_size,
         array_name=None,
+        compressor=None,
+        filters=None,
     ):
-        shape = [num_variants]
         prefix = "variant_"
         dimensions = ["variants"]
-        chunks = [variants_chunk_size]
         if vcf_field.category == "FORMAT":
             prefix = "call_"
-            shape.append(num_samples)
-            chunks.append(samples_chunk_size)
             dimensions.append("samples")
         if array_name is None:
             array_name = prefix + vcf_field.name
-        # TODO make an option to add in the empty extra dimension
-        if vcf_field.summary.max_number > 1 or vcf_field.full_name == "FORMAT/LAA":
-            shape.append(vcf_field.summary.max_number)
-            chunks.append(vcf_field.summary.max_number)
-            # TODO we should really be checking this to see if the named dimensions
-            # are actually correct.
-            if vcf_field.vcf_number == "R":
+
+        max_number = vcf_field.max_number
+        if vcf_field.vcf_number == "R":
+            max_alleles = schema.dimensions["alleles"].size
+            if max_number > max_alleles:
+                raise ValueError(
+                    f"Max number of values {max_number} exceeds max alleles "
+                    f"{max_alleles} for {vcf_field.full_name}"
+                )
+            if max_alleles > 0:
                 dimensions.append("alleles")
-            elif vcf_field.vcf_number == "A":
+        elif vcf_field.vcf_number == "A":
+            max_alt_alleles = schema.dimensions["alt_alleles"].size
+            if max_number > max_alt_alleles:
+                raise ValueError(
+                    f"Max number of values {max_number} exceeds max alt alleles "
+                    f"{max_alt_alleles} for {vcf_field.full_name}"
+                )
+            if max_alt_alleles > 0:
                 dimensions.append("alt_alleles")
-            elif vcf_field.vcf_number == "G":
+        elif vcf_field.vcf_number == "G":
+            max_genotypes = schema.dimensions["genotypes"].size
+            if max_number > max_genotypes:
+                raise ValueError(
+                    f"Max number of values {max_number} exceeds max genotypes "
+                    f"{max_genotypes} for {vcf_field.full_name}"
+                )
+            if max_genotypes > 0:
                 dimensions.append("genotypes")
-            else:
-                dimensions.append(f"{vcf_field.category}_{vcf_field.name}_dim")
-        return ZarrArraySpec.new(
-            vcf_field=vcf_field.full_name,
+        elif max_number > 1 or vcf_field.full_name == "FORMAT/LAA":
+            dimensions.append(f"{vcf_field.category}_{vcf_field.name}_dim")
+        if dimensions[-1] not in schema.dimensions:
+            schema.dimensions[dimensions[-1]] = VcfZarrDimension.unchunked(
+                vcf_field.max_number
+            )
+
+        return ZarrArraySpec(
+            source=vcf_field.full_name,
             name=array_name,
             dtype=vcf_field.smallest_dtype(),
-            shape=shape,
-            chunks=chunks,
             dimensions=dimensions,
             description=vcf_field.description,
+            compressor=compressor,
+            filters=filters,
         )
 
-    def _choose_compressor_settings(self):
-        """
-        Choose compressor and filter settings based on the size and
-        type of the array, plus some hueristics from observed properties
-        of VCFs.
-
-        See https://github.com/pystatgen/bio2zarr/discussions/74
-        """
-        # Default is to not shuffle, because autoshuffle isn't recognised
-        # by many Zarr implementations, and shuffling can lead to worse
-        # performance in some cases anyway. Turning on shuffle should be a
-        # deliberate choice.
-        shuffle = numcodecs.Blosc.NOSHUFFLE
-        if self.name == "call_genotype" and self.dtype == "i1":
-            # call_genotype gets BITSHUFFLE by default as it gets
-            # significantly better compression (at a cost of slower
-            # decoding)
-            shuffle = numcodecs.Blosc.BITSHUFFLE
-        elif self.dtype == "bool":
-            shuffle = numcodecs.Blosc.BITSHUFFLE
-
-        self.compressor["shuffle"] = shuffle
-
-    @property
-    def chunk_nbytes(self):
+    def chunk_nbytes(self, schema):
         """
         Returns the nbytes for a single chunk in this array.
         """
         items = 1
         dim = 0
-        for chunk_size in self.chunks:
-            size = min(chunk_size, self.shape[dim])
+        for chunk_size in self.get_chunks(schema):
+            size = min(chunk_size, self.get_shape(schema)[dim])
             items *= size
             dim += 1
         # Include sizes for extra dimensions.
-        for size in self.shape[dim:]:
+        for size in self.get_shape(schema)[dim:]:
             items *= size
         dt = np.dtype(self.dtype)
         return items * dt.itemsize
 
-    @property
-    def variant_chunk_nbytes(self):
+    def variant_chunk_nbytes(self, schema):
         """
         Returns the nbytes for a single variant chunk of this array.
         """
-        chunk_items = self.chunks[0]
-        for size in self.shape[1:]:
+        chunk_items = self.get_chunks(schema)[0]
+        for size in self.get_shape(schema)[1:]:
             chunk_items *= size
         dt = np.dtype(self.dtype)
-        if dt.kind == "O" and "samples" in self.dimensions:
+        if dt.kind == zarr_utils.STRING_DTYPE_NAME and "samples" in self.dimensions:
             logger.warning(
                 f"Field {self.name} is a string; max memory usage may "
                 "be a significant underestimate"
@@ -181,87 +292,71 @@ class ZarrArraySpec:
         return chunk_items * dt.itemsize
 
 
-ZARR_SCHEMA_FORMAT_VERSION = "0.4"
+@dataclasses.dataclass
+class Contig:
+    id: str
+    length: int = None
 
 
-def convert_local_allele_field_types(fields):
-    """
-    Update the specified list of fields to include the LAA field, and to convert
-    any supported localisable fields to the L* counterpart.
-
-    Note that we currently support only two ALT alleles per sample, and so the
-    dimensions of these fields are fixed by that requirement. Later versions may
-    use summry data storted in the ICF to make different choices, if information
-    about subsequent alleles (not in the actual genotype calls) should also be
-    stored.
-    """
-    fields_by_name = {field.name: field for field in fields}
-    gt = fields_by_name["call_genotype"]
-    if gt.shape[-1] != 2:
-        raise ValueError("Local alleles only supported on diploid data")
-
-    # TODO check if LA is already in here
-
-    shape = gt.shape[:-1]
-    chunks = gt.chunks[:-1]
-    dimensions = gt.dimensions[:-1]
-
-    la = ZarrArraySpec.new(
-        vcf_field=None,
-        name="call_LA",
-        dtype="i1",
-        shape=gt.shape,
-        chunks=gt.chunks,
-        dimensions=(*dimensions, "local_alleles"),
-        description=(
-            "0-based indices into REF+ALT, indicating which alleles"
-            " are relevant (local) for the current sample"
-        ),
-    )
-    ad = fields_by_name.get("call_AD", None)
-    if ad is not None:
-        # TODO check if call_LAD is in the list already
-        ad.name = "call_LAD"
-        ad.vcf_field = None
-        ad.shape = (*shape, 2)
-        ad.chunks = (*chunks, 2)
-        ad.dimensions = (*dimensions, "local_alleles")
-        ad.description += " (local-alleles)"
-
-    pl = fields_by_name.get("call_PL", None)
-    if pl is not None:
-        # TODO check if call_LPL is in the list already
-        pl.name = "call_LPL"
-        pl.vcf_field = None
-        pl.shape = (*shape, 3)
-        pl.chunks = (*chunks, 3)
-        pl.description += " (local-alleles)"
-        pl.dimensions = (*dimensions, "local_" + pl.dimensions[-1])
-    return [*fields, la]
+@dataclasses.dataclass
+class Sample:
+    id: str
+
+
+@dataclasses.dataclass
+class Filter:
+    id: str
+    description: str = ""
 
 
 @dataclasses.dataclass
 class VcfZarrSchema(core.JsonDataclass):
     format_version: str
-    samples_chunk_size: int
-    variants_chunk_size: int
-    samples: list
-    contigs: list
-    filters: list
+    dimensions: dict
     fields: list
+    defaults: dict
+
+    def __init__(
+        self,
+        format_version: str,
+        fields: list,
+        dimensions: dict,
+        defaults: dict = None,
+    ):
+        self.format_version = format_version
+        self.fields = fields
+        defaults = defaults.copy() if defaults is not None else {}
+        if defaults.get("compressor", None) is None:
+            defaults["compressor"] = DEFAULT_ZARR_COMPRESSOR.get_config()
+        if defaults.get("filters", None) is None:
+            defaults["filters"] = []
+        self.defaults = defaults
+        self.dimensions = dimensions
+
+    def get_shape(self, dimensions):
+        return [self.dimensions[dim].size for dim in dimensions]
+
+    def get_chunks(self, dimensions):
+        return [self.dimensions[dim].chunk_size for dim in dimensions]
 
     def validate(self):
         """
         Checks that the schema is well-formed and within required limits.
         """
         for field in self.fields:
+            for dim in field.dimensions:
+                if dim not in self.dimensions:
+                    raise ValueError(
+                        f"Dimension '{dim}' used in field '{field.name}' is "
+                        "not defined in the schema"
+                    )
+
+            chunk_nbytes = field.get_chunk_nbytes(self)
             # This is the Blosc max buffer size
-            if field.chunk_nbytes > 2147483647:
-                # TODO add some links to documentation here advising how to
-                # deal with PL values.
+            if chunk_nbytes > 2147483647:
                 raise ValueError(
                     f"Field {field.name} chunks are too large "
-                    f"({field.chunk_nbytes} > 2**31 - 1 bytes). "
+                    f"({chunk_nbytes} > 2**31 - 1 bytes). "
                     "Either generate a schema and drop this field (if you don't "
                     "need it) or reduce the variant or sample chunk sizes."
                 )
@@ -278,253 +373,30 @@ class VcfZarrSchema(core.JsonDataclass):
                 "Zarr schema format version mismatch: "
                 f"{d['format_version']} != {ZARR_SCHEMA_FORMAT_VERSION}"
             )
+
         ret = VcfZarrSchema(**d)
-        ret.samples = [icf.Sample(**sd) for sd in d["samples"]]
-        ret.contigs = [icf.Contig(**sd) for sd in d["contigs"]]
-        ret.filters = [icf.Filter(**sd) for sd in d["filters"]]
         ret.fields = [ZarrArraySpec(**sd) for sd in d["fields"]]
+        ret.dimensions = {
+            k: VcfZarrDimension.fromdict(v) for k, v in d["dimensions"].items()
+        }
+
         return ret
 
     @staticmethod
     def fromjson(s):
         return VcfZarrSchema.fromdict(json.loads(s))
 
-    @staticmethod
-    def generate(
-        icf, variants_chunk_size=None, samples_chunk_size=None, local_alleles=None
-    ):
-        m = icf.num_records
-        n = icf.num_samples
-        if samples_chunk_size is None:
-            samples_chunk_size = 10_000
-        if variants_chunk_size is None:
-            variants_chunk_size = 1000
-        if local_alleles is None:
-            local_alleles = False
-        logger.info(
-            f"Generating schema with chunks={variants_chunk_size, samples_chunk_size}"
-        )
-
-        def spec_from_field(field, array_name=None):
-            return ZarrArraySpec.from_field(
-                field,
-                num_samples=n,
-                num_variants=m,
-                samples_chunk_size=samples_chunk_size,
-                variants_chunk_size=variants_chunk_size,
-                array_name=array_name,
-            )
 
-        def fixed_field_spec(
-            name,
-            dtype,
-            vcf_field=None,
-            shape=(m,),
-            dimensions=("variants",),
-            chunks=None,
-        ):
-            return ZarrArraySpec.new(
-                vcf_field=vcf_field,
-                name=name,
-                dtype=dtype,
-                shape=shape,
-                description="",
-                dimensions=dimensions,
-                chunks=chunks or [variants_chunk_size],
-            )
-
-        alt_field = icf.fields["ALT"]
-        max_alleles = alt_field.vcf_field.summary.max_number + 1
-
-        array_specs = [
-            fixed_field_spec(
-                name="variant_contig",
-                dtype=core.min_int_dtype(0, icf.metadata.num_contigs),
-            ),
-            fixed_field_spec(
-                name="variant_filter",
-                dtype="bool",
-                shape=(m, icf.metadata.num_filters),
-                dimensions=["variants", "filters"],
-                chunks=(variants_chunk_size, icf.metadata.num_filters),
-            ),
-            fixed_field_spec(
-                name="variant_allele",
-                dtype="O",
-                shape=(m, max_alleles),
-                dimensions=["variants", "alleles"],
-                chunks=(variants_chunk_size, max_alleles),
-            ),
-            fixed_field_spec(
-                name="variant_id",
-                dtype="O",
-            ),
-            fixed_field_spec(
-                name="variant_id_mask",
-                dtype="bool",
-            ),
-        ]
-        name_map = {field.full_name: field for field in icf.metadata.fields}
-
-        # Only three of the fixed fields have a direct one-to-one mapping.
-        array_specs.extend(
-            [
-                spec_from_field(name_map["QUAL"], array_name="variant_quality"),
-                spec_from_field(name_map["POS"], array_name="variant_position"),
-                spec_from_field(name_map["rlen"], array_name="variant_length"),
-            ]
-        )
-        array_specs.extend(
-            [spec_from_field(field) for field in icf.metadata.info_fields]
-        )
-
-        gt_field = None
-        for field in icf.metadata.format_fields:
-            if field.name == "GT":
-                gt_field = field
-                continue
-            array_specs.append(spec_from_field(field))
-
-        if gt_field is not None and n > 0:
-            ploidy = max(gt_field.summary.max_number - 1, 1)
-            shape = [m, n]
-            chunks = [variants_chunk_size, samples_chunk_size]
-            dimensions = ["variants", "samples"]
-            array_specs.append(
-                ZarrArraySpec.new(
-                    vcf_field=None,
-                    name="call_genotype_phased",
-                    dtype="bool",
-                    shape=list(shape),
-                    chunks=list(chunks),
-                    dimensions=list(dimensions),
-                    description="",
-                )
-            )
-            shape += [ploidy]
-            chunks += [ploidy]
-            dimensions += ["ploidy"]
-            array_specs.append(
-                ZarrArraySpec.new(
-                    vcf_field=None,
-                    name="call_genotype",
-                    dtype=gt_field.smallest_dtype(),
-                    shape=list(shape),
-                    chunks=list(chunks),
-                    dimensions=list(dimensions),
-                    description="",
-                )
-            )
-            array_specs.append(
-                ZarrArraySpec.new(
-                    vcf_field=None,
-                    name="call_genotype_mask",
-                    dtype="bool",
-                    shape=list(shape),
-                    chunks=list(chunks),
-                    dimensions=list(dimensions),
-                    description="",
-                )
-            )
-
-        if local_alleles:
-            array_specs = convert_local_allele_field_types(array_specs)
-
-        return VcfZarrSchema(
-            format_version=ZARR_SCHEMA_FORMAT_VERSION,
-            samples_chunk_size=samples_chunk_size,
-            variants_chunk_size=variants_chunk_size,
-            fields=array_specs,
-            samples=icf.metadata.samples,
-            contigs=icf.metadata.contigs,
-            filters=icf.metadata.filters,
-        )
-
-
-class VcfZarr:
-    def __init__(self, path):
-        if not (path / ".zmetadata").exists():
-            raise ValueError("Not in VcfZarr format")  # NEEDS TEST
-        self.path = path
-        self.root = zarr.open(path, mode="r")
-
-    def summary_table(self):
-        data = []
-        arrays = [(core.du(self.path / a.basename), a) for _, a in self.root.arrays()]
-        arrays.sort(key=lambda x: x[0])
-        for stored, array in reversed(arrays):
-            d = {
-                "name": array.name,
-                "dtype": str(array.dtype),
-                "stored": core.display_size(stored),
-                "size": core.display_size(array.nbytes),
-                "ratio": core.display_number(array.nbytes / stored),
-                "nchunks": str(array.nchunks),
-                "chunk_size": core.display_size(array.nbytes / array.nchunks),
-                "avg_chunk_stored": core.display_size(int(stored / array.nchunks)),
-                "shape": str(array.shape),
-                "chunk_shape": str(array.chunks),
-                "compressor": str(array.compressor),
-                "filters": str(array.filters),
-            }
-            data.append(d)
-        return data
-
-
-def parse_max_memory(max_memory):
-    if max_memory is None:
-        # Effectively unbounded
-        return 2**63
-    if isinstance(max_memory, str):
-        max_memory = humanfriendly.parse_size(max_memory)
-    logger.info(f"Set memory budget to {core.display_size(max_memory)}")
-    return max_memory
-
-
-@dataclasses.dataclass
-class VcfZarrPartition:
-    start: int
-    stop: int
-
-    @staticmethod
-    def generate_partitions(num_records, chunk_size, num_partitions, max_chunks=None):
-        num_chunks = int(np.ceil(num_records / chunk_size))
-        if max_chunks is not None:
-            num_chunks = min(num_chunks, max_chunks)
-        partitions = []
-        splits = np.array_split(np.arange(num_chunks), min(num_partitions, num_chunks))
-        for chunk_slice in splits:
-            start_chunk = int(chunk_slice[0])
-            stop_chunk = int(chunk_slice[-1]) + 1
-            start_index = start_chunk * chunk_size
-            stop_index = min(stop_chunk * chunk_size, num_records)
-            partitions.append(VcfZarrPartition(start_index, stop_index))
-        return partitions
-
-
-VZW_METADATA_FORMAT_VERSION = "0.1"
-
-
-@dataclasses.dataclass
-class VcfZarrWriterMetadata(core.JsonDataclass):
-    format_version: str
-    icf_path: str
-    schema: VcfZarrSchema
-    dimension_separator: str
-    partitions: list
-    provenance: dict
-
-    @staticmethod
-    def fromdict(d):
-        if d["format_version"] != VZW_METADATA_FORMAT_VERSION:
-            raise ValueError(
-                "VcfZarrWriter format version mismatch: "
-                f"{d['format_version']} != {VZW_METADATA_FORMAT_VERSION}"
-            )
-        ret = VcfZarrWriterMetadata(**d)
-        ret.schema = VcfZarrSchema.fromdict(ret.schema)
-        ret.partitions = [VcfZarrPartition(**p) for p in ret.partitions]
-        return ret
+def sanitise_int_array(value, ndmin, dtype):
+    if isinstance(value, tuple):
+        value = [
+            constants.VCF_INT_MISSING if x is None else x for x in value
+        ]  # NEEDS TEST
+    value = np.array(value, ndmin=ndmin, copy=True)
+    value[value == constants.VCF_INT_MISSING] = -1
+    value[value == constants.VCF_INT_FILL] = -2
+    # TODO watch out for clipping here!
+    return value.astype(dtype)
 
 
 def compute_la_field(genotypes):
@@ -597,14 +469,60 @@ class LocalisableFieldDescriptor:
 
 localisable_fields = [
     LocalisableFieldDescriptor(
-        "call_LAD", "FORMAT/AD", icf.sanitise_int_array, compute_lad_field
+        "call_LAD", "FORMAT/AD", sanitise_int_array, compute_lad_field
     ),
     LocalisableFieldDescriptor(
-        "call_LPL", "FORMAT/PL", icf.sanitise_int_array, compute_lpl_field
+        "call_LPL", "FORMAT/PL", sanitise_int_array, compute_lpl_field
     ),
 ]
 
 
+@dataclasses.dataclass
+class VcfZarrPartition:
+    start: int
+    stop: int
+
+    @staticmethod
+    def generate_partitions(num_records, chunk_size, num_partitions, max_chunks=None):
+        num_chunks = int(np.ceil(num_records / chunk_size))
+        if max_chunks is not None:
+            num_chunks = min(num_chunks, max_chunks)
+        partitions = []
+        splits = np.array_split(np.arange(num_chunks), min(num_partitions, num_chunks))
+        for chunk_slice in splits:
+            start_chunk = int(chunk_slice[0])
+            stop_chunk = int(chunk_slice[-1]) + 1
+            start_index = start_chunk * chunk_size
+            stop_index = min(stop_chunk * chunk_size, num_records)
+            partitions.append(VcfZarrPartition(start_index, stop_index))
+        return partitions
+
+
+VZW_METADATA_FORMAT_VERSION = "0.1"
+
+
+@dataclasses.dataclass
+class VcfZarrWriterMetadata(core.JsonDataclass):
+    format_version: str
+    source_path: str
+    schema: VcfZarrSchema
+    dimension_separator: str
+    partitions: list
+    provenance: dict
+
+    @staticmethod
+    def fromdict(d):
+        if d["format_version"] != VZW_METADATA_FORMAT_VERSION:
+            raise ValueError(
+                "VcfZarrWriter format version mismatch: "
+                f"{d['format_version']} != {VZW_METADATA_FORMAT_VERSION}"
+            )
+        ret = VcfZarrWriterMetadata(**d)
+        ret.schema = VcfZarrSchema.fromdict(ret.schema)
+        ret.partitions = [VcfZarrPartition(**p) for p in ret.partitions]
+        return ret
+
+
 @dataclasses.dataclass
 class VcfZarrWriteSummary(core.JsonDataclass):
     num_partitions: int
@@ -615,13 +533,14 @@ class VcfZarrWriteSummary(core.JsonDataclass):
 
 
 class VcfZarrWriter:
-    def __init__(self, path):
+    def __init__(self, source_type, path):
+        self.source_type = source_type
         self.path = pathlib.Path(path)
         self.wip_path = self.path / "wip"
         self.arrays_path = self.wip_path / "arrays"
         self.partitions_path = self.wip_path / "partitions"
         self.metadata = None
-        self.icf = None
+        self.source = None
 
     @property
     def schema(self):
@@ -639,7 +558,7 @@ class VcfZarrWriter:
 
     def has_local_alleles(self):
         for field in self.schema.fields:
-            if field.name == "call_LA" and field.vcf_field is None:
+            if field.name == "call_LA" and field.source is None:
                 return True
         return False
 
@@ -649,20 +568,20 @@ class VcfZarrWriter:
 
     def init(
         self,
-        icf,
+        source,
         *,
         target_num_partitions,
         schema,
         dimension_separator=None,
         max_variant_chunks=None,
     ):
-        self.icf = icf
+        self.source = source
         if self.path.exists():
             raise ValueError("Zarr path already exists")  # NEEDS TEST
         schema.validate()
         partitions = VcfZarrPartition.generate_partitions(
-            self.icf.num_records,
-            schema.variants_chunk_size,
+            self.source.num_records,
+            schema.get_chunks(["variants"])[0],
             target_num_partitions,
             max_chunks=max_variant_chunks,
         )
@@ -673,7 +592,7 @@ class VcfZarrWriter:
         )
         self.metadata = VcfZarrWriterMetadata(
             format_version=VZW_METADATA_FORMAT_VERSION,
-            icf_path=str(self.icf.path),
+            source_path=str(self.source.path),
             schema=schema,
             dimension_separator=dimension_separator,
             partitions=partitions,
@@ -682,27 +601,32 @@ class VcfZarrWriter:
         )
 
         self.path.mkdir()
-        root = zarr.open(store=self.path, mode="a", **ZARR_FORMAT_KWARGS)
+        root = zarr.open(store=self.path, mode="a", **zarr_utils.ZARR_FORMAT_KWARGS)
         root.attrs.update(
             {
-                "vcf_zarr_version": "0.2",
-                "vcf_header": self.icf.vcf_header,
+                "vcf_zarr_version": "0.4",
                 "source": f"bio2zarr-{provenance.__version__}",
             }
         )
-        # Doing this syncronously - this is fine surely
+        root.attrs.update(self.source.root_attrs)
+
+        # Doing this synchronously - this is fine surely
         self.encode_samples(root)
-        self.encode_filter_id(root)
-        self.encode_contig_id(root)
+        if self.source.filters is not None:
+            self.encode_filters(root)
+        if self.source.contigs is not None:
+            self.encode_contigs(root)
 
         self.wip_path.mkdir()
         self.arrays_path.mkdir()
         self.partitions_path.mkdir()
-        root = zarr.open(store=self.arrays_path, mode="a", **ZARR_FORMAT_KWARGS)
+        root = zarr.open(
+            store=self.arrays_path, mode="a", **zarr_utils.ZARR_FORMAT_KWARGS
+        )
 
         total_chunks = 0
         for field in self.schema.fields:
-            a = self.init_array(root, field, partitions[-1].stop)
+            a = self.init_array(root, self.metadata.schema, field, partitions[-1].stop)
             total_chunks += a.nchunks
 
         logger.info("Writing WIP metadata")
@@ -710,89 +634,111 @@ class VcfZarrWriter:
             json.dump(self.metadata.asdict(), f, indent=4)
 
         return VcfZarrWriteSummary(
-            num_variants=self.icf.num_records,
-            num_samples=self.icf.num_samples,
+            num_variants=self.source.num_records,
+            num_samples=self.source.num_samples,
             num_partitions=self.num_partitions,
             num_chunks=total_chunks,
             max_encoding_memory=core.display_size(self.get_max_encoding_memory()),
         )
 
     def encode_samples(self, root):
-        if self.schema.samples != self.icf.metadata.samples:
-            raise ValueError("Subsetting or reordering samples not supported currently")
-        array = root.array(
+        samples = self.source.samples
+        zarr_utils.create_group_array(
+            root,
             "sample_id",
-            data=[sample.id for sample in self.schema.samples],
-            shape=len(self.schema.samples),
+            data=[sample.id for sample in samples],
+            shape=len(samples),
             dtype="str",
             compressor=DEFAULT_ZARR_COMPRESSOR,
-            chunks=(self.schema.samples_chunk_size,),
+            chunks=(self.schema.get_chunks(["samples"])[0],),
+            dimension_names=["samples"],
         )
-        array.attrs["_ARRAY_DIMENSIONS"] = ["samples"]
         logger.debug("Samples done")
 
-    def encode_contig_id(self, root):
-        array = root.array(
+    def encode_contigs(self, root):
+        contigs = self.source.contigs
+        zarr_utils.create_group_array(
+            root,
             "contig_id",
-            data=[contig.id for contig in self.schema.contigs],
-            shape=len(self.schema.contigs),
+            data=[contig.id for contig in contigs],
+            shape=len(contigs),
             dtype="str",
             compressor=DEFAULT_ZARR_COMPRESSOR,
+            dimension_names=["contigs"],
         )
-        array.attrs["_ARRAY_DIMENSIONS"] = ["contigs"]
-        if all(contig.length is not None for contig in self.schema.contigs):
-            array = root.array(
+        if all(contig.length is not None for contig in contigs):
+            zarr_utils.create_group_array(
+                root,
                 "contig_length",
-                data=[contig.length for contig in self.schema.contigs],
-                shape=len(self.schema.contigs),
+                data=[contig.length for contig in contigs],
+                shape=len(contigs),
                 dtype=np.int64,
                 compressor=DEFAULT_ZARR_COMPRESSOR,
+                dimension_names=["contigs"],
             )
-            array.attrs["_ARRAY_DIMENSIONS"] = ["contigs"]
 
-    def encode_filter_id(self, root):
-        # TODO need a way to store description also
-        # https://github.com/sgkit-dev/vcf-zarr-spec/issues/19
-        array = root.array(
+    def encode_filters(self, root):
+        filters = self.source.filters
+        zarr_utils.create_group_array(
+            root,
             "filter_id",
-            data=[filt.id for filt in self.schema.filters],
-            shape=len(self.schema.filters),
+            data=[filt.id for filt in filters],
+            shape=len(filters),
+            dtype="str",
+            compressor=DEFAULT_ZARR_COMPRESSOR,
+            dimension_names=["filters"],
+        )
+        zarr_utils.create_group_array(
+            root,
+            "filter_description",
+            data=[filt.description for filt in filters],
+            shape=len(filters),
             dtype="str",
             compressor=DEFAULT_ZARR_COMPRESSOR,
+            dimension_names=["filters"],
         )
-        array.attrs["_ARRAY_DIMENSIONS"] = ["filters"]
 
-    def init_array(self, root, array_spec, variants_dim_size):
-        kwargs = dict(ZARR_FORMAT_KWARGS)
-        filters = [numcodecs.get_codec(filt) for filt in array_spec.filters]
-        if array_spec.dtype == "O":
-            if zarr_v3():
+    def init_array(self, root, schema, array_spec, variants_dim_size):
+        kwargs = dict(zarr_utils.ZARR_FORMAT_KWARGS)
+        filters = (
+            array_spec.filters
+            if array_spec.filters is not None
+            else schema.defaults["filters"]
+        )
+        filters = [numcodecs.get_codec(filt) for filt in filters]
+        compressor = (
+            array_spec.compressor
+            if array_spec.compressor is not None
+            else schema.defaults["compressor"]
+        )
+        compressor = numcodecs.get_codec(compressor)
+        if array_spec.dtype == zarr_utils.STRING_DTYPE_NAME:
+            if zarr_utils.zarr_v3():
                 filters = [*list(filters), numcodecs.VLenUTF8()]
             else:
                 kwargs["object_codec"] = numcodecs.VLenUTF8()
 
-        if not zarr_v3():
+        if zarr_utils.zarr_v3():
+            # see https://github.com/zarr-developers/zarr-python/issues/3197
+            kwargs["fill_value"] = None
+        else:
             kwargs["dimension_separator"] = self.metadata.dimension_separator
 
-        shape = list(array_spec.shape)
-        # Truncate the variants dimension is max_variant_chunks was specified
+        shape = schema.get_shape(array_spec.dimensions)
+        # Truncate the variants dimension if max_variant_chunks was specified
         shape[0] = variants_dim_size
-        a = root.empty(
+        a = zarr_utils.create_empty_group_array(
+            root,
             name=array_spec.name,
             shape=shape,
-            chunks=array_spec.chunks,
+            chunks=schema.get_chunks(array_spec.dimensions),
             dtype=array_spec.dtype,
-            compressor=numcodecs.get_codec(array_spec.compressor),
+            compressor=compressor,
             filters=filters,
+            dimension_names=array_spec.dimensions,
             **kwargs,
         )
-        a.attrs.update(
-            {
-                "description": array_spec.description,
-                # Dimension names are part of the spec in Zarr v3
-                "_ARRAY_DIMENSIONS": array_spec.dimensions,
-            }
-        )
+        a.attrs.update({"description": array_spec.description})
         logger.debug(f"Initialised {a}")
         return a
 
@@ -804,7 +750,7 @@ class VcfZarrWriter:
         if self.metadata is None:
             with open(self.wip_path / "metadata.json") as f:
                 self.metadata = VcfZarrWriterMetadata.fromdict(json.load(f))
-            self.icf = icf.IntermediateColumnarFormat(self.metadata.icf_path)
+            self.source = self.source_type(self.metadata.source_path)
 
     def partition_path(self, partition_index):
         return self.partitions_path / f"p{partition_index}"
@@ -826,15 +772,18 @@ class VcfZarrWriter:
         partition_path.mkdir(exist_ok=True)
         logger.info(f"Encoding partition {partition_index} to {partition_path}")
 
-        self.encode_id_partition(partition_index)
-        self.encode_filters_partition(partition_index)
-        self.encode_contig_partition(partition_index)
-        self.encode_alleles_partition(partition_index)
+        all_field_names = [field.name for field in self.schema.fields]
+        if "variant_id" in all_field_names:
+            self.encode_id_partition(partition_index)
+        if "variant_filter" in all_field_names:
+            self.encode_filters_partition(partition_index)
+        if "variant_contig" in all_field_names:
+            self.encode_contig_partition(partition_index)
+        self.encode_alleles_and_genotypes_partition(partition_index)
         for array_spec in self.schema.fields:
-            if array_spec.vcf_field is not None:
+            if array_spec.source is not None:
                 self.encode_array_partition(array_spec, partition_index)
         if self.has_genotypes():
-            self.encode_genotypes_partition(partition_index)
             self.encode_genotype_mask_partition(partition_index)
         if self.has_local_alleles():
             self.encode_local_alleles_partition(partition_index)
@@ -874,34 +823,48 @@ class VcfZarrWriter:
     def encode_array_partition(self, array_spec, partition_index):
         partition = self.metadata.partitions[partition_index]
         ba = self.init_partition_array(partition_index, array_spec.name)
-        source_field = self.icf.fields[array_spec.vcf_field]
-        sanitiser = source_field.sanitiser_factory(ba.buff.shape)
-
-        for value in source_field.iter_values(partition.start, partition.stop):
-            # We write directly into the buffer in the sanitiser function
-            # to make it easier to reason about dimension padding
+        for value in self.source.iter_field(
+            array_spec.source,
+            ba.buff.shape[1:],
+            partition.start,
+            partition.stop,
+        ):
             j = ba.next_buffer_row()
-            sanitiser(ba.buff, j, value)
+            ba.buff[j] = value
+
         self.finalise_partition_array(partition_index, ba)
 
-    def encode_genotypes_partition(self, partition_index):
+    def encode_alleles_and_genotypes_partition(self, partition_index):
         partition = self.metadata.partitions[partition_index]
-        gt = self.init_partition_array(partition_index, "call_genotype")
-        gt_phased = self.init_partition_array(partition_index, "call_genotype_phased")
-
-        source_field = self.icf.fields["FORMAT/GT"]
-        for value in source_field.iter_values(partition.start, partition.stop):
-            j = gt.next_buffer_row()
-            icf.sanitise_value_int_2d(
-                gt.buff, j, value[:, :-1] if value is not None else None
-            )
-            j = gt_phased.next_buffer_row()
-            icf.sanitise_value_int_1d(
-                gt_phased.buff, j, value[:, -1] if value is not None else None
+        alleles = self.init_partition_array(partition_index, "variant_allele")
+        variant_lengths = self.init_partition_array(partition_index, "variant_length")
+        has_gt = self.has_genotypes()
+        shape = None
+        if has_gt:
+            gt = self.init_partition_array(partition_index, "call_genotype")
+            gt_phased = self.init_partition_array(
+                partition_index, "call_genotype_phased"
             )
+            shape = gt.buff.shape[1:]
 
-        self.finalise_partition_array(partition_index, gt)
-        self.finalise_partition_array(partition_index, gt_phased)
+        for variant_data in self.source.iter_alleles_and_genotypes(
+            partition.start, partition.stop, shape, alleles.array.shape[1]
+        ):
+            j_alleles = alleles.next_buffer_row()
+            alleles.buff[j_alleles] = variant_data.alleles
+            j_variant_length = variant_lengths.next_buffer_row()
+            variant_lengths.buff[j_variant_length] = variant_data.variant_length
+            if has_gt:
+                j = gt.next_buffer_row()
+                gt.buff[j] = variant_data.genotypes
+                j_phased = gt_phased.next_buffer_row()
+                gt_phased.buff[j_phased] = variant_data.phased
+
+        self.finalise_partition_array(partition_index, alleles)
+        self.finalise_partition_array(partition_index, variant_lengths)
+        if has_gt:
+            self.finalise_partition_array(partition_index, gt)
+            self.finalise_partition_array(partition_index, gt_phased)
 
     def encode_genotype_mask_partition(self, partition_index):
         partition = self.metadata.partitions[partition_index]
@@ -948,10 +911,10 @@ class VcfZarrWriter:
         for descriptor in localisable_fields:
             if descriptor.array_name not in field_map:
                 continue
-            assert field_map[descriptor.array_name].vcf_field is None
+            assert field_map[descriptor.array_name].source is None
 
             buff = self.init_partition_array(partition_index, descriptor.array_name)
-            source = self.icf.fields[descriptor.vcf_field].iter_values(
+            source = self.source.fields[descriptor.vcf_field].iter_values(
                 partition.start, partition.stop
             )
             for la in core.first_dim_slice_iter(
@@ -963,34 +926,17 @@ class VcfZarrWriter:
                 buff.buff[j] = descriptor.convert(value, la)
             self.finalise_partition_array(partition_index, buff)
 
-    def encode_alleles_partition(self, partition_index):
-        alleles = self.init_partition_array(partition_index, "variant_allele")
-        partition = self.metadata.partitions[partition_index]
-        ref_field = self.icf.fields["REF"]
-        alt_field = self.icf.fields["ALT"]
-
-        for ref, alt in zip(
-            ref_field.iter_values(partition.start, partition.stop),
-            alt_field.iter_values(partition.start, partition.stop),
-        ):
-            j = alleles.next_buffer_row()
-            alleles.buff[j, :] = constants.STR_FILL
-            alleles.buff[j, 0] = ref[0]
-            alleles.buff[j, 1 : 1 + len(alt)] = alt
-        self.finalise_partition_array(partition_index, alleles)
-
     def encode_id_partition(self, partition_index):
         vid = self.init_partition_array(partition_index, "variant_id")
         vid_mask = self.init_partition_array(partition_index, "variant_id_mask")
         partition = self.metadata.partitions[partition_index]
-        field = self.icf.fields["ID"]
 
-        for value in field.iter_values(partition.start, partition.stop):
+        for value in self.source.iter_id(partition.start, partition.stop):
             j = vid.next_buffer_row()
             k = vid_mask.next_buffer_row()
             assert j == k
             if value is not None:
-                vid.buff[j] = value[0]
+                vid.buff[j] = value
                 vid_mask.buff[j] = False
             else:
                 vid.buff[j] = constants.STR_MISSING
@@ -1000,37 +946,22 @@ class VcfZarrWriter:
         self.finalise_partition_array(partition_index, vid_mask)
 
     def encode_filters_partition(self, partition_index):
-        lookup = {filt.id: index for index, filt in enumerate(self.schema.filters)}
         var_filter = self.init_partition_array(partition_index, "variant_filter")
         partition = self.metadata.partitions[partition_index]
 
-        field = self.icf.fields["FILTERS"]
-        for value in field.iter_values(partition.start, partition.stop):
+        for filter_values in self.source.iter_filters(partition.start, partition.stop):
             j = var_filter.next_buffer_row()
-            var_filter.buff[j] = False
-            for f in value:
-                try:
-                    var_filter.buff[j, lookup[f]] = True
-                except KeyError:
-                    raise ValueError(
-                        f"Filter '{f}' was not defined in the header."
-                    ) from None
+            var_filter.buff[j] = filter_values
 
         self.finalise_partition_array(partition_index, var_filter)
 
     def encode_contig_partition(self, partition_index):
-        lookup = {contig.id: index for index, contig in enumerate(self.schema.contigs)}
         contig = self.init_partition_array(partition_index, "variant_contig")
         partition = self.metadata.partitions[partition_index]
-        field = self.icf.fields["CHROM"]
 
-        for value in field.iter_values(partition.start, partition.stop):
+        for contig_index in self.source.iter_contig(partition.start, partition.stop):
             j = contig.next_buffer_row()
-            # Note: because we are using the indexes to define the lookups
-            # and we always have an index, it seems that we the contig lookup
-            # will always succeed. However, if anyone ever does hit a KeyError
-            # here, please do open an issue with a reproducible example!
-            contig.buff[j] = lookup[value[0]]
+            contig.buff[j] = contig_index
 
         self.finalise_partition_array(partition_index, contig)
 
@@ -1050,19 +981,7 @@ class VcfZarrWriter:
             if not src.exists():
                 # Needs test
                 raise ValueError(f"Partition {partition} of {name} does not exist")
-            dest = self.arrays_path / name
-            # This is Zarr v2 specific. Chunks in v3 with start with "c" prefix.
-            chunk_files = [
-                path for path in src.iterdir() if not path.name.startswith(".")
-            ]
-            # TODO check for a count of then number of files. If we require a
-            # dimension_separator of "/" then we could make stronger assertions
-            # here, as we'd always have num_variant_chunks
-            logger.debug(
-                f"Moving {len(chunk_files)} chunks for {name} partition {partition}"
-            )
-            for chunk_file in chunk_files:
-                os.rename(chunk_file, dest / chunk_file.name)
+            zarr_utils.move_chunks(src, self.arrays_path, partition, name)
         # Finally, once all the chunks have moved into the arrays dir,
         # we move it out of wip
         os.rename(self.arrays_path / name, self.path / name)
@@ -1109,60 +1028,8 @@ class VcfZarrWriter:
     def create_index(self):
         """Create an index to support efficient region queries."""
 
-        root = zarr.open_group(store=self.path, mode="r+")
-
-        contig = root["variant_contig"]
-        pos = root["variant_position"]
-        length = root["variant_length"]
-
-        assert contig.cdata_shape == pos.cdata_shape
-
-        index = []
-
-        logger.info("Creating region index")
-        for v_chunk in range(pos.cdata_shape[0]):
-            c = contig.blocks[v_chunk]
-            p = pos.blocks[v_chunk]
-            e = p + length.blocks[v_chunk] - 1
-
-            # create a row for each contig in the chunk
-            d = np.diff(c, append=-1)
-            c_start_idx = 0
-            for c_end_idx in np.nonzero(d)[0]:
-                assert c[c_start_idx] == c[c_end_idx]
-                index.append(
-                    (
-                        v_chunk,  # chunk index
-                        c[c_start_idx],  # contig ID
-                        p[c_start_idx],  # start
-                        p[c_end_idx],  # end
-                        np.max(e[c_start_idx : c_end_idx + 1]),  # max end
-                        c_end_idx - c_start_idx + 1,  # num records
-                    )
-                )
-                c_start_idx = c_end_idx + 1
-
-        index = np.array(index, dtype=pos.dtype)
-        kwargs = {}
-        if not zarr_v3():
-            kwargs["dimension_separator"] = self.metadata.dimension_separator
-        array = root.array(
-            "region_index",
-            data=index,
-            shape=index.shape,
-            chunks=index.shape,
-            dtype=index.dtype,
-            compressor=numcodecs.Blosc("zstd", clevel=9, shuffle=0),
-            fill_value=None,
-            **kwargs,
-        )
-        array.attrs["_ARRAY_DIMENSIONS"] = [
-            "region_index_values",
-            "region_index_fields",
-        ]
-
-        logger.info("Consolidating Zarr metadata")
-        zarr.consolidate_metadata(self.path)
+        indexer = VcfZarrIndexer(self.path)
+        indexer.create_index()
 
     ######################
     # encode_all_partitions
@@ -1174,11 +1041,13 @@ class VcfZarrWriter:
         """
         max_encoding_mem = 0
         for array_spec in self.schema.fields:
-            max_encoding_mem = max(max_encoding_mem, array_spec.variant_chunk_nbytes)
+            max_encoding_mem = max(
+                max_encoding_mem, array_spec.variant_chunk_nbytes(self.schema)
+            )
         gt_mem = 0
         if self.has_genotypes:
             gt_mem = sum(
-                field.variant_chunk_nbytes
+                field.variant_chunk_nbytes(self.schema)
                 for field in self.schema.fields
                 if field.name.startswith("call_genotype")
             )
@@ -1187,7 +1056,7 @@ class VcfZarrWriter:
     def encode_all_partitions(
         self, *, worker_processes=1, show_progress=False, max_memory=None
     ):
-        max_memory = parse_max_memory(max_memory)
+        max_memory = core.parse_max_memory(max_memory)
         self.load_metadata()
         num_partitions = self.num_partitions
         per_worker_memory = self.get_max_encoding_memory()
@@ -1229,147 +1098,107 @@ class VcfZarrWriter:
                 pwm.submit(self.encode_partition, partition_index)
 
 
-def mkschema(
-    if_path,
-    out,
-    *,
-    variants_chunk_size=None,
-    samples_chunk_size=None,
-    local_alleles=None,
-):
-    store = icf.IntermediateColumnarFormat(if_path)
-    spec = VcfZarrSchema.generate(
-        store,
-        variants_chunk_size=variants_chunk_size,
-        samples_chunk_size=samples_chunk_size,
-        local_alleles=local_alleles,
-    )
-    out.write(spec.asjson())
-
-
-def encode(
-    if_path,
-    zarr_path,
-    schema_path=None,
-    variants_chunk_size=None,
-    samples_chunk_size=None,
-    max_variant_chunks=None,
-    dimension_separator=None,
-    max_memory=None,
-    local_alleles=None,
-    worker_processes=1,
-    show_progress=False,
-):
-    # Rough heuristic to split work up enough to keep utilisation high
-    target_num_partitions = max(1, worker_processes * 4)
-    encode_init(
-        if_path,
-        zarr_path,
-        target_num_partitions,
-        schema_path=schema_path,
-        variants_chunk_size=variants_chunk_size,
-        samples_chunk_size=samples_chunk_size,
-        local_alleles=local_alleles,
-        max_variant_chunks=max_variant_chunks,
-        dimension_separator=dimension_separator,
-    )
-    vzw = VcfZarrWriter(zarr_path)
-    vzw.encode_all_partitions(
-        worker_processes=worker_processes,
-        show_progress=show_progress,
-        max_memory=max_memory,
-    )
-    vzw.finalise(show_progress)
-    vzw.create_index()
-
-
-def encode_init(
-    icf_path,
-    zarr_path,
-    target_num_partitions,
-    *,
-    schema_path=None,
-    variants_chunk_size=None,
-    samples_chunk_size=None,
-    local_alleles=None,
-    max_variant_chunks=None,
-    dimension_separator=None,
-    max_memory=None,
-    worker_processes=1,
-    show_progress=False,
-):
-    icf_store = icf.IntermediateColumnarFormat(icf_path)
-    if schema_path is None:
-        schema = VcfZarrSchema.generate(
-            icf_store,
-            variants_chunk_size=variants_chunk_size,
-            samples_chunk_size=samples_chunk_size,
-            local_alleles=local_alleles,
-        )
-    else:
-        logger.info(f"Reading schema from {schema_path}")
-        if variants_chunk_size is not None or samples_chunk_size is not None:
+class VcfZarr:
+    def __init__(self, path):
+        if not zarr_utils.zarr_exists(path):
+            raise ValueError("Not in VcfZarr format")  # NEEDS TEST
+        self.path = path
+        self.root = zarr.open(path, mode="r")
+
+    def summary_table(self):
+        data = []
+        arrays = [(core.du(self.path / a.basename), a) for _, a in self.root.arrays()]
+        arrays.sort(key=lambda x: x[0])
+        for stored, array in reversed(arrays):
+            d = {
+                "name": array.name,
+                "dtype": str(array.dtype),
+                "stored": core.display_size(stored),
+                "size": core.display_size(array.nbytes),
+                "ratio": core.display_number(array.nbytes / stored),
+                "nchunks": str(array.nchunks),
+                "chunk_size": core.display_size(array.nbytes / array.nchunks),
+                "avg_chunk_stored": core.display_size(int(stored / array.nchunks)),
+                "shape": str(array.shape),
+                "chunk_shape": str(array.chunks),
+                "compressor": str(zarr_utils.get_compressor(array)),
+                "filters": str(array.filters),
+            }
+            data.append(d)
+        return data
+
+
+class VcfZarrIndexer:
+    """
+    Creates an index for efficient region queries in a VCF Zarr dataset.
+    """
+
+    def __init__(self, path):
+        self.path = pathlib.Path(path)
+
+    def create_index(self):
+        """Create an index to support efficient region queries."""
+        root = zarr.open_group(store=self.path, mode="r+")
+        if (
+            "variant_contig" not in root
+            or "variant_position" not in root
+            or "variant_length" not in root
+        ):
             raise ValueError(
-                "Cannot specify schema along with chunk sizes"
-            )  # NEEDS TEST
-        with open(schema_path) as f:
-            schema = VcfZarrSchema.fromjson(f.read())
-    zarr_path = pathlib.Path(zarr_path)
-    vzw = VcfZarrWriter(zarr_path)
-    return vzw.init(
-        icf_store,
-        target_num_partitions=target_num_partitions,
-        schema=schema,
-        dimension_separator=dimension_separator,
-        max_variant_chunks=max_variant_chunks,
-    )
-
-
-def encode_partition(zarr_path, partition):
-    writer = VcfZarrWriter(zarr_path)
-    writer.encode_partition(partition)
-
-
-def encode_finalise(zarr_path, show_progress=False):
-    writer = VcfZarrWriter(zarr_path)
-    writer.finalise(show_progress=show_progress)
-
-
-def convert(
-    vcfs,
-    out_path,
-    *,
-    variants_chunk_size=None,
-    samples_chunk_size=None,
-    worker_processes=1,
-    local_alleles=None,
-    show_progress=False,
-    icf_path=None,
-):
-    if icf_path is None:
-        cm = temp_icf_path(prefix="vcf2zarr")
-    else:
-        cm = contextlib.nullcontext(icf_path)
-
-    with cm as icf_path:
-        icf.explode(
-            icf_path,
-            vcfs,
-            worker_processes=worker_processes,
-            show_progress=show_progress,
-        )
-        encode(
-            icf_path,
-            out_path,
-            variants_chunk_size=variants_chunk_size,
-            samples_chunk_size=samples_chunk_size,
-            worker_processes=worker_processes,
-            show_progress=show_progress,
-            local_alleles=local_alleles,
-        )
+                "Cannot create index: variant_contig, "
+                "variant_position and variant_length arrays are required"
+            )
+
+        contig = root["variant_contig"]
+        pos = root["variant_position"]
+        length = root["variant_length"]
+
+        assert contig.cdata_shape == pos.cdata_shape
 
+        index = []
+
+        logger.info("Creating region index")
+        for v_chunk in range(pos.cdata_shape[0]):
+            c = contig.blocks[v_chunk]
+            p = pos.blocks[v_chunk]
+            e = p + length.blocks[v_chunk] - 1
 
-@contextlib.contextmanager
-def temp_icf_path(prefix=None):
-    with tempfile.TemporaryDirectory(prefix=prefix) as tmp:
-        yield pathlib.Path(tmp) / "icf"
+            # create a row for each contig in the chunk
+            d = np.diff(c, append=-1)
+            c_start_idx = 0
+            for c_end_idx in np.nonzero(d)[0]:
+                assert c[c_start_idx] == c[c_end_idx]
+                index.append(
+                    (
+                        v_chunk,  # chunk index
+                        c[c_start_idx],  # contig ID
+                        p[c_start_idx],  # start
+                        p[c_end_idx],  # end
+                        np.max(e[c_start_idx : c_end_idx + 1]),  # max end
+                        c_end_idx - c_start_idx + 1,  # num records
+                    )
+                )
+                c_start_idx = c_end_idx + 1
+
+        index = np.array(index, dtype=pos.dtype)
+        kwargs = {}
+        if not zarr_utils.zarr_v3():
+            kwargs["dimension_separator"] = "/"
+        zarr_utils.create_group_array(
+            root,
+            "region_index",
+            data=index,
+            shape=index.shape,
+            chunks=index.shape,
+            dtype=index.dtype,
+            compressor=numcodecs.Blosc("zstd", clevel=9, shuffle=0),
+            fill_value=None,
+            dimension_names=[
+                "region_index_values",
+                "region_index_fields",
+            ],
+            **kwargs,
+        )
+
+        logger.info("Consolidating Zarr metadata")
+        zarr.consolidate_metadata(self.path)