bio2zarr 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

bio2zarr/{vcf2zarr/verification.py → vcz_verification.py}

@@ -1,12 +1,12 @@
-import cyvcf2
 import numpy as np
 import numpy.testing as nt
 import tqdm
 import zarr
 
+from bio2zarr import core
 from bio2zarr.zarr_utils import first_dim_iter
 
-from .. import constants
+from . import constants
 
 
 def assert_all_missing_float(a):
@@ -146,7 +146,10 @@ def assert_format_val_equal(vcf_val, zarr_val, vcf_type, vcf_number):
         nt.assert_equal(vcf_val, zarr_val)
 
 
+@core.requires_optional_dependency("cyvcf2", "vcf")
 def verify(vcf_path, zarr_path, show_progress=False):
+    import cyvcf2
+
     root = zarr.open(store=zarr_path, mode="r")
     pos = root["variant_position"][:]
     allele = root["variant_allele"][:]

bio2zarr/zarr_utils.py

@@ -1,18 +1,185 @@
+import logging
+import os
+
 import zarr
 
+logger = logging.getLogger(__name__)
+
+# Use zarr format v2 by default even when running with zarr-python v3
+# NOTE: this interface was introduced for experimentation with zarr
+# format 3 and is not envisaged as a long-term interface.
+try:
+    ZARR_FORMAT = int(os.environ.get("BIO2ZARR_ZARR_FORMAT", "2"))
+except Exception:
+    ZARR_FORMAT = 2
+
 
 def zarr_v3() -> bool:
     return zarr.__version__ >= "3"
 
 
 if zarr_v3():
-    # Use zarr format v2 even when running with zarr-python v3
-    ZARR_FORMAT_KWARGS = dict(zarr_format=2)
+    ZARR_FORMAT_KWARGS = dict(zarr_format=ZARR_FORMAT)
+    # In zarr-python v3 strings are stored as string arrays (T) with itemsize 16
+    STRING_DTYPE_NAME = "T"
+    STRING_ITEMSIZE = 16
 else:
     ZARR_FORMAT_KWARGS = dict()
+    # In zarr-python v2 strings are stored as object arrays (O) with itemsize 8
+    STRING_DTYPE_NAME = "O"
+    STRING_ITEMSIZE = 8
 
 
 # See discussion in https://github.com/zarr-developers/zarr-python/issues/2529
 def first_dim_iter(z):
     for chunk in range(z.cdata_shape[0]):
         yield from z.blocks[chunk]
+
+
+def zarr_exists(path):
+    # NOTE: this is too strict, we should support more general Zarrs, see #276
+    return (path / ".zmetadata").exists() or (path / "zarr.json").exists()
+
+
+def create_group_array(
+    group,
+    name,
+    *,
+    data,
+    shape,
+    dtype,
+    compressor=None,
+    dimension_names=None,
+    **kwargs,
+):
+    """Create an array within a group."""
+    if ZARR_FORMAT == 2:
+        array = group.array(
+            name,
+            data=data,
+            shape=shape,
+            dtype=dtype,
+            compressor=compressor,
+            **kwargs,
+        )
+        if dimension_names is not None:
+            array.attrs["_ARRAY_DIMENSIONS"] = dimension_names
+        return array
+    else:
+        new_kwargs = {**kwargs}
+        if compressor is not None:
+            compressors = [_convert_v2_compressor_to_v3_codec(compressor, dtype)]
+            # TODO: seems odd that we need to set this
+            new_kwargs["compressor"] = "auto"
+            new_kwargs["compressors"] = compressors
+        return group.array(
+            name,
+            data=data,
+            shape=shape,
+            dtype=dtype,
+            dimension_names=dimension_names,
+            **new_kwargs,
+        )
+
+
+def create_empty_group_array(
+    group,
+    name,
+    *,
+    shape,
+    dtype,
+    chunks,
+    compressor=None,
+    filters=None,
+    dimension_names=None,
+    **kwargs,
+):
+    """Create an empty array within a group."""
+    if ZARR_FORMAT == 2:
+        array = group.empty(
+            name=name,
+            shape=shape,
+            dtype=dtype,
+            chunks=chunks,
+            compressor=compressor,
+            filters=filters,
+            **kwargs,
+        )
+        if dimension_names is not None:
+            array.attrs["_ARRAY_DIMENSIONS"] = dimension_names
+        return array
+    else:
+        new_kwargs = {**kwargs}
+        new_kwargs.pop("zarr_format")
+        if compressor is not None:
+            compressors = [_convert_v2_compressor_to_v3_codec(compressor, dtype)]
+            # TODO: seems odd that we need to set this
+            new_kwargs["compressor"] = "auto"
+            new_kwargs["compressors"] = compressors
+        return group.array(
+            name=name,
+            shape=shape,
+            dtype=dtype,
+            chunks=chunks,
+            dimension_names=dimension_names,
+            **new_kwargs,
+        )
+
+
+def get_compressor(array):
+    try:
+        # zarr format v2: compressor (singular)
+        return array.compressor
+    except TypeError as e:
+        # zarr format v3: compressors (plural)
+        compressors = array.compressors
+        if len(compressors) > 1:
+            raise ValueError(
+                f"Only one compressor is supported but found {compressors}"
+            ) from e
+        return compressors[0] if len(compressors) == 1 else None
+
+
+def get_compressor_config(array):
+    compressor = get_compressor(array)
+    if hasattr(compressor, "get_config"):
+        return compressor.get_config()
+    else:
+        from zarr.codecs.blosc import BloscCodec
+
+        if isinstance(compressor, BloscCodec):
+            return compressor._blosc_codec.get_config()
+        else:
+            return compressor.as_dict()["configuration"]
+
+
+def _convert_v2_compressor_to_v3_codec(compressor, dtype):
+    # import here since this is zarr-python v3 only
+    from zarr.core.dtype import parse_dtype
+    from zarr.metadata.migrate_v3 import _convert_compressor
+
+    return _convert_compressor(compressor, parse_dtype(dtype, zarr_format=3))
+
+
+def move_chunks(src_path, dest_path, partition, name):
+    if ZARR_FORMAT == 2:
+        dest = dest_path / name
+        chunk_files = [
+            path for path in src_path.iterdir() if not path.name.startswith(".")
+        ]
+    else:
+        dest = dest_path / name / "c"
+        dest.mkdir(exist_ok=True)
+        src_chunks = src_path / "c"
+        if not src_chunks.exists():
+            chunk_files = []
+        else:
+            chunk_files = [
+                path for path in src_chunks.iterdir() if not path.name.startswith(".")
+            ]
+    # TODO check for a count of then number of files. If we require a
+    # dimension_separator of "/" then we could make stronger assertions
+    # here, as we'd always have num_variant_chunks
+    logger.debug(f"Moving {len(chunk_files)} chunks for {name} partition {partition}")
+    for chunk_file in chunk_files:
+        os.rename(chunk_file, dest / chunk_file.name)

{bio2zarr-0.1.5.dist-info → bio2zarr-0.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bio2zarr
-Version: 0.1.5
+Version: 0.1.7
 Summary: Convert bioinformatics data to Zarr
 Author-email: sgkit Developers <project@sgkit.dev>
 License:                                  Apache License
@@ -216,35 +216,50 @@ Classifier: Operating System :: MacOS :: MacOS X
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: numpy>=1.26
+Requires-Dist: numpy>=2
 Requires-Dist: zarr<3,>=2.17
-Requires-Dist: click
+Requires-Dist: numcodecs[msgpack]!=0.14.0,!=0.14.1,<0.16
 Requires-Dist: tabulate
 Requires-Dist: tqdm
 Requires-Dist: humanfriendly
-Requires-Dist: cyvcf2
-Requires-Dist: bed_reader
+Requires-Dist: coloredlogs
+Requires-Dist: click
+Requires-Dist: pandas
 Provides-Extra: dev
+Requires-Dist: click>=8.2.0; extra == "dev"
 Requires-Dist: hypothesis-vcf; extra == "dev"
 Requires-Dist: msprime; extra == "dev"
 Requires-Dist: pysam; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-coverage; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: sgkit>=0.8.0; extra == "dev"
 Requires-Dist: tqdm; extra == "dev"
+Requires-Dist: tskit>=1; extra == "dev"
+Requires-Dist: bed_reader; extra == "dev"
+Requires-Dist: cyvcf2; extra == "dev"
+Requires-Dist: xarray<2025.03.1; extra == "dev"
+Requires-Dist: dask[array]<=2024.8.0,>=2022.01.0; extra == "dev"
+Provides-Extra: tskit
+Requires-Dist: tskit>=1; extra == "tskit"
+Provides-Extra: vcf
+Requires-Dist: cyvcf2; extra == "vcf"
+Provides-Extra: all
+Requires-Dist: tskit>=1; extra == "all"
+Requires-Dist: cyvcf2; extra == "all"
 Dynamic: license-file
 
 [![CI](https://github.com/sgkit-dev/bio2zarr/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/sgkit-dev/bio2zarr/actions/workflows/ci.yml)
 [![Coverage Status](https://coveralls.io/repos/github/sgkit-dev/bio2zarr/badge.svg)](https://coveralls.io/github/sgkit-dev/bio2zarr)
+[![PyPI Downloads](https://static.pepy.tech/badge/bio2zarr)](https://pepy.tech/projects/bio2zarr)
+[![Anaconda-Server Badge](https://anaconda.org/bioconda/bio2zarr/badges/downloads.svg)](https://anaconda.org/bioconda/bio2zarr)
 
 
 # bio2zarr

bio2zarr-0.1.7.dist-info/RECORD

@@ -0,0 +1,21 @@
+bio2zarr/__init__.py,sha256=KiUGyya-9RHNcBldB8Lc1g3rP3CRjaL-5Olben0_6qA,49
+bio2zarr/__main__.py,sha256=4pF1IBO4CcswA_Fe7NmK_pqGOUHCwsd_8YU7dP92n9c,578
+bio2zarr/_version.py,sha256=szvPIs2C82UunpzuvVg3MbF4QhzbBYTsVJ8DmPfq6_E,704
+bio2zarr/cli.py,sha256=iHfmc-qU2roQXm9Bt3TyR2bmgH-2p3DqYosQERePMZ8,17873
+bio2zarr/constants.py,sha256=QjbtFeBUZ-XqG35ZFIFj8EYrta_EwUkC2B5VGRP7oQs,425
+bio2zarr/core.py,sha256=mYi2Vmh_YdNEd3weE0zZIPr7ToEUynq8nNCVvONVaqM,12140
+bio2zarr/plink.py,sha256=ELGhsSdH1Xmxx6agCfTx1kYyntrU0XQ384wxTEn87BM,11717
+bio2zarr/provenance.py,sha256=c_Z__QbWkLS0Rfa8D7LgEhtStng_zRMJX8comaDXIkw,142
+bio2zarr/tskit.py,sha256=iLheNWtX7Pad1oNfijf6THMphzXwEtuQ6Zmi94pRZHg,10847
+bio2zarr/typing.py,sha256=HdXNwIBEqYtGNwKyeUDQv6-H-pKSwNZO0qD2_VxTXEY,48
+bio2zarr/vcf.py,sha256=3aXCdTAIuGoUmpbPIPVKhNj4oevkF0s_l7gRB0QmaPU,60738
+bio2zarr/vcf_utils.py,sha256=xrsmxpu1xyXtl6FaYuU562WZP-iVUIaqzxD-11MHfAM,19541
+bio2zarr/vcz.py,sha256=3IkcrAsQkWCiHiMBh0bbxzHtvX8qaUV3W84y1ojUWSs,42204
+bio2zarr/vcz_verification.py,sha256=4YZZnAuMH-z9uPqAeBONdsZADz2MtY57D7RAbMa90yY,8119
+bio2zarr/zarr_utils.py,sha256=4vE6CqnOLqZExc_7Z0jGGbA-kjqz9NPSqSBue10bzHk,5443
+bio2zarr-0.1.7.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+bio2zarr-0.1.7.dist-info/METADATA,sha256=wXANeYEuZh41wH_nay96e4xobWhpBhL-BzkBcdGAR04,15736
+bio2zarr-0.1.7.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+bio2zarr-0.1.7.dist-info/entry_points.txt,sha256=bbIbR8fWMGruyLaoCxO1O22nKidWKUzMgYbTYdsN6YQ,181
+bio2zarr-0.1.7.dist-info/top_level.txt,sha256=ouAvp3u9N25eKrQbN8BCDLPcWWQLhtlgdHKu8AtEj5Q,9
+bio2zarr-0.1.7.dist-info/RECORD,,

{bio2zarr-0.1.5.dist-info → bio2zarr-0.1.7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{bio2zarr-0.1.5.dist-info → bio2zarr-0.1.7.dist-info}/entry_points.txt

@@ -1,3 +1,5 @@
 [console_scripts]
+plink2zarr = bio2zarr.cli:plink2zarr_main
+tskit2zarr = bio2zarr.cli:tskit2zarr_main
 vcf2zarr = bio2zarr.cli:vcf2zarr_main
 vcfpartition = bio2zarr.cli:vcfpartition

bio2zarr/vcf2zarr/__init__.py

@@ -1,38 +0,0 @@
-from .icf import (
-    IntermediateColumnarFormat,
-    explode,
-    explode_finalise,
-    explode_init,
-    explode_partition,
-)
-from .vcz import (
-    VcfZarrSchema,
-    convert,
-    encode,
-    encode_finalise,
-    encode_init,
-    encode_partition,
-    inspect,
-    mkschema,
-)
-from .verification import verify
-
-# NOTE some of these aren't intended to be part of the external
-# interface (like IntermediateColumnarFormat), but putting
-# them into the list to keep the lint nagging under control
-__all__ = [
-    "IntermediateColumnarFormat",
-    "explode",
-    "explode_finalise",
-    "explode_init",
-    "explode_partition",
-    "VcfZarrSchema",
-    "convert",
-    "encode",
-    "encode_finalise",
-    "encode_init",
-    "encode_partition",
-    "inspect",
-    "mkschema",
-    "verify",
-]

bio2zarr-0.1.5.dist-info/RECORD

@@ -1,21 +0,0 @@
-bio2zarr/__init__.py,sha256=KiUGyya-9RHNcBldB8Lc1g3rP3CRjaL-5Olben0_6qA,49
-bio2zarr/__main__.py,sha256=wUKNNps8MAAEpMvLgVaI449eKyfr7Jpk2mMtYbNl4Ek,531
-bio2zarr/_version.py,sha256=Y4jy4bEMmwl_qNPCmiMFnlQ2ofMoqyG37hp8uwI3m10,511
-bio2zarr/cli.py,sha256=eyOSqU7hlZuvXEVB2g3qWPK6ys0A1C1gMahVz51hRqs,15999
-bio2zarr/constants.py,sha256=QjbtFeBUZ-XqG35ZFIFj8EYrta_EwUkC2B5VGRP7oQs,425
-bio2zarr/core.py,sha256=4xqNf3Txgyhcx23bzXZHq3GW0Jh24fPQwob7lKO7s0w,11668
-bio2zarr/plink.py,sha256=Yr1meT4AgS2qnwM64-Nmthh4HbjaPXsddYiJdtfYWBg,6999
-bio2zarr/provenance.py,sha256=c_Z__QbWkLS0Rfa8D7LgEhtStng_zRMJX8comaDXIkw,142
-bio2zarr/typing.py,sha256=BYxhL16sKRoNxa6amf6AYxvt5Ke9qzv2np_kOT_zPJo,79
-bio2zarr/vcf_utils.py,sha256=u1nkFRecY__IgkfV3N0Sr3AFIUSN8sYEF463K1HIgEE,19496
-bio2zarr/zarr_utils.py,sha256=99J7ycaG92K_AcWRF2S9A4ec2_4cXL6kjYT99GBfli4,415
-bio2zarr/vcf2zarr/__init__.py,sha256=0_of1iGzIDhvti49Gbcgd47oP63mKvouk9uLgKgiwoQ,791
-bio2zarr/vcf2zarr/icf.py,sha256=G70eC6LgrJUvGBHKYrcV83BA7Mm3D170zIsoXRZgoUA,42895
-bio2zarr/vcf2zarr/vcz.py,sha256=cfUCBsQW5dbhDu7NzXkd1Dalsev7UkFDXVOyChAHw8Q,49409
-bio2zarr/vcf2zarr/verification.py,sha256=uM-mg0yvUTBs-MvWBd4jxTS0zKCUbxEQpm4ALJADdMI,8037
-bio2zarr-0.1.5.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-bio2zarr-0.1.5.dist-info/METADATA,sha256=rWYid_erOvB8gywz8N4TXBfR7ezSELuaF5Hyq3iV86w,15000
-bio2zarr-0.1.5.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-bio2zarr-0.1.5.dist-info/entry_points.txt,sha256=3adtRrClMpjatEbiYqK5bm9WHA2PaJN5hK-Cs_zkpaI,97
-bio2zarr-0.1.5.dist-info/top_level.txt,sha256=ouAvp3u9N25eKrQbN8BCDLPcWWQLhtlgdHKu8AtEj5Q,9
-bio2zarr-0.1.5.dist-info/RECORD,,