bed-reader 1.0.0b1__tar.gz → 1.0.0b4__tar.gz

{bed_reader-1.0.0b1 → bed_reader-1.0.0b4}/.github/workflows/ci.yml

@@ -170,6 +170,8 @@ jobs:
         virtualenv vEnv; source vEnv/bin/activate
         python -m pip install --upgrade pip maturin
         maturin develop
+        pip install -r requirements-min-dev.txt
+        pytest bed_reader/tests/test_opt_dep.py
         echo "requirements"
         pip install -r requirements.txt
         echo "requirements-dev"
@@ -183,6 +185,8 @@ jobs:
         echo "pwd"
         pwd
         python -m pip install --upgrade pip maturin
+        pip install -r requirements-min-dev.txt
+        pytest bed_reader/tests/test_opt_dep.py
         echo "requirements"
         pip install -r requirements.txt
         echo "requirements-dev"

{bed_reader-1.0.0b1 → bed_reader-1.0.0b4}/CHANGELOG.md

@@ -5,17 +5,17 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.0.0] - 2023-11-04
+## [1.0.0] - 2023-11-5
 
 ### Changed
 
-- (Python) Install now with `pip install bed-reader[samples,sparse]`.
-  If you want a version that depends only `numpy` (and for now pandas) use
-  `pip install bed-reader`.
+- (Python) Core project now depends only on `numpy`.
+  If you want to download samples files or create
+  sparse matrices, say `pip install bed-reader[samples,sparse]`.
 
 - (Python) Added new `read_sparse()` method for creating
   `sympy.sparse.csc_matrix` and `sympy.sparse.csr_matrix`. This
-  method can save memory when a *.bed file contains mostly 0's.
+  method can save memory when a *.bed file contains mostly zeros.
 
 - (Rust) Rust methods that returned `Result` now return
   `Result<_,Box<BedErrorPlus>>`. Before, they returned

{bed_reader-1.0.0b1 → bed_reader-1.0.0b4}/Cargo.lock

@@ -86,7 +86,7 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
 [[package]]
 name = "bed-reader"
-version = "1.0.0-beta.1"
+version = "1.0.0-beta.4"
 dependencies = [
  "anyinput",
  "approx 0.5.1",

{bed_reader-1.0.0b1 → bed_reader-1.0.0b4}/Cargo.toml

@@ -1,7 +1,7 @@
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package]
-version = "1.0.0-beta.1"
+version = "1.0.0-beta.4"
 name = "bed-reader"
 description = "Read and write the PLINK BED format, simply and efficiently."
 repository = "https://github.com/fastlmm/bed-reader"

{bed_reader-1.0.0b1 → bed_reader-1.0.0b4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bed-reader
-Version: 1.0.0b1
+Version: 1.0.0b4
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
@@ -9,7 +9,6 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python
 Requires-Dist: numpy >=1.13.3
-Requires-Dist: pandas >=0.25.1
 Requires-Dist: pooch >=1.5.0 ; extra == 'samples'
 Requires-Dist: scipy >=1.4 ; extra == 'sparse'
 Provides-Extra: samples
@@ -28,11 +27,12 @@ Project-URL: Source Code, https://github.com/fastlmm/bed-reader
 Project-URL: Bug Tracker, https://github.com/fastlmm/bed-reader/issues
 Project-URL: Documentation, http://fastlmm.github.io/bed-reader
 
-<!-- This must be keep in sync with doc/source/index.rst manually -->
+<!-- This README.md must be keep in sync with doc/source/index.rst manually -->
 
 <!-- markdownlint-disable MD046 -->
+<!-- MD046 is about 'code block style' -->
 <!-- markdownlint-disable MD041 -->
-[![PyPI version](https://badge.fury.io/py/bed-reader.svg)](https://badge.fury.io/py/bed-reader)
+<!-- MD041 is about 'first line in a file should be a top-level heading'. -->
 [![PyPI version](https://badge.fury.io/py/bed-reader.svg)](https://badge.fury.io/py/bed-reader)
 [![Build Status](https://github.com/fastlmm/bed-reader/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/fastlmm/bed-reader/actions/workflows/ci.yml)
 ![PyPI](https://img.shields.io/pypi/v/bed-reader)

{bed_reader-1.0.0b1 → bed_reader-1.0.0b4}/README.md

@@ -1,8 +1,9 @@
-<!-- This must be keep in sync with doc/source/index.rst manually -->
+<!-- This README.md must be keep in sync with doc/source/index.rst manually -->
 
 <!-- markdownlint-disable MD046 -->
+<!-- MD046 is about 'code block style' -->
 <!-- markdownlint-disable MD041 -->
-[![PyPI version](https://badge.fury.io/py/bed-reader.svg)](https://badge.fury.io/py/bed-reader)
+<!-- MD041 is about 'first line in a file should be a top-level heading'. -->
 [![PyPI version](https://badge.fury.io/py/bed-reader.svg)](https://badge.fury.io/py/bed-reader)
 [![Build Status](https://github.com/fastlmm/bed-reader/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/fastlmm/bed-reader/actions/workflows/ci.yml)
 ![PyPI](https://img.shields.io/pypi/v/bed-reader)

{bed_reader-1.0.0b1 → bed_reader-1.0.0b4}/bed_reader/_open_bed.py

@@ -13,9 +13,6 @@ try:
 except ImportError:
     sparse = None
 
-# cmk remove pandas
-import pandas as pd
-
 from .bed_reader import read_f32, read_f64, read_i8  # type: ignore
 
 
@@ -1146,9 +1143,6 @@ class open_bed:
         delimiter = _delimiters[suffix]
         if delimiter in {r"\s+"}:
             delimiter = None
-            delim_whitespace = True
-        else:
-            delim_whitespace = False
 
         usecolsdict = {}
         dtype_dict = {}
@@ -1160,34 +1154,30 @@ class open_bed:
         assert len(usecolsdict) > 0  # real assert
 
         if os.path.getsize(property_filepath) == 0:
-            fields = []
+            columns, row_count = [], 0
         else:
-            fields = pd.read_csv(
+            columns, row_count = _read_csv(
                 property_filepath,
                 delimiter=delimiter,
-                delim_whitespace=delim_whitespace,
-                header=None,
-                index_col=False,
-                comment=None,
                 dtype=dtype_dict,
                 usecols=usecolsdict.values(),
             )
 
         if count is None:
-            self._counts[suffix] = len(fields)
+            self._counts[suffix] = row_count
         else:
-            if count != len(fields):
+            if count != row_count:
                 raise ValueError(
-                    f"The number of lines in the *.{suffix} file, {len(fields)}, "
+                    f"The number of lines in the *.{suffix} file, {row_count}, "
                     + "should not be different from the current "
                     + "f{_count_name[suffix]}, {count}"
                 )
-        for key in usecolsdict.keys():
+        for i, key in enumerate(usecolsdict.keys()):
             mm = _meta_meta[key]
-            if len(fields) == 0:
+            if row_count == 0:
                 output = np.array([], dtype=mm.dtype)
             else:
-                output = fields[mm.column].values
+                output = columns[i]
                 if not np.issubdtype(output.dtype, mm.dtype):
                     output = np.array(output, dtype=mm.dtype)
             self.properties_dict[key] = output
@@ -1202,7 +1192,14 @@ class open_bed:
     ) -> (Union[sparse.csc_matrix, sparse.csr_matrix]) if sparse is not None else None:
         """
         Read genotype information into a :mod:`scipy.sparse` matrix. Sparse matrices
-        may be useful when the data is mostly 0.
+        may be useful when the data is mostly zeros.
+
+        .. note::
+            This method requires :mod:`scipy`. Install `scipy` with:
+
+            .. code-block:: bash
+
+                pip install --upgrade bed-reader[sparse]
 
         Parameters
         ----------
@@ -1367,7 +1364,7 @@ class open_bed:
             or len(sid_index) > np.iinfo(np.int32).max
         ):
             raise ValueError(
-                "Too many Individuals or SNPs (variants) requested. Maximum is {np.iinfo(np.int32).max}."
+                "Too (many Individuals or SNPs (variants) requested. Maximum is {np.iinfo(np.int32).max}."
             )
 
         if batch_size is None:
@@ -1493,9 +1490,60 @@ class open_bed:
         indptr[1:][batch_slice] = np.cumsum(counts_with_initial)[1:]
 
 
-if __name__ == "__main__":
-    logging.basicConfig(level=logging.INFO)
+def _read_csv(filepath, delimiter=None, dtype=None, usecols=None):
+    # Prepare the usecols by ensuring it is a list of indices
+    usecols_indices = list(usecols)
+    transposed = np.loadtxt(
+        filepath,
+        dtype=np.str_,
+        delimiter=delimiter,
+        usecols=usecols_indices,
+        unpack=True,
+    )
+    if transposed.ndim == 1:
+        transposed = transposed.reshape(-1, 1)
+    row_count = transposed.shape[1]  # because unpack=True
+
+    # Convert column lists to numpy arrays with the specified dtype
+    columns = []
+    for output_index, input_index in enumerate(usecols_indices):
+        col = transposed[output_index]
+        # Find the dtype for this column
+        col_dtype = dtype.get(input_index, np.str_)
+        # Convert the column list to a numpy array with the specified dtype
+        columns.append(_convert_to_dtype(col, col_dtype))
+
+    return columns, row_count
+
+
+def _convert_to_dtype(str_arr, dtype):
+    assert dtype in [np.str_, np.float32, np.int32]  # real assert
+
+    if dtype == np.str_:
+        return str_arr
+
+    try:
+        new_arr = str_arr.astype(dtype)
+    except ValueError as e:
+        if dtype == np.float32:
+            raise e
+        # for backwards compatibility, see if intermediate float helps int conversion
+        try:
+            assert dtype == np.int32  # real assert
+            float_arr = str_arr.astype(np.float32)
+        except ValueError:
+            raise e
+        new_arr = float_arr.astype(np.int32)
+        if not np.array_equal(new_arr, float_arr):
+            raise ValueError(
+                f"invalid literal for int: '{str_arr[np.where(new_arr != float_arr)][:1]}')"
+            )
+    return new_arr
+
 
+if __name__ == "__main__":
     import pytest
 
+    logging.basicConfig(level=logging.INFO)
+
     pytest.main(["--doctest-modules", __file__])

{bed_reader-1.0.0b1 → bed_reader-1.0.0b4}/bed_reader/_sample_data.py

@@ -44,6 +44,14 @@ def sample_file(filepath: Union[str, Path]) -> str:
         Local name of sample .bed file.
 
 
+    .. note::
+        This function requires the :mod:`pooch` package. Install `pooch` with:
+
+        .. code-block:: bash
+
+            pip install --upgrade bed-reader[samples]
+
+
     By default this function puts files under the user's cache directory.
     Override this by setting
     the `BED_READER_DATA_DIR` environment variable.

{bed_reader-1.0.0b1 → bed_reader-1.0.0b4}/bed_reader/tests/test_open_bed.py

@@ -5,12 +5,9 @@ from pathlib import Path
 
 import numpy as np
 import pytest
-import scipy.sparse as sparse
 
 from bed_reader import open_bed, subset_f64_f64, to_bed
 
-# cmk somehow add test that get nice message for no pooch or scipy.
-
 
 def test_read1(shared_datadir):
     import math
@@ -413,6 +410,8 @@ def test_write1_x_x_cpp(tmp_path, shared_datadir):
 
 
 def test_respect_read_inputs(shared_datadir):
+    import scipy.sparse as sparse
+
     ref_val_float = reference_val(shared_datadir)
     ref_val_float2 = ref_val_float.copy()
     ref_val_float2[ref_val_float != ref_val_float] = -127
@@ -899,6 +898,39 @@ def test_sparse():
         print(val_sparse.shape)
 
 
+def test_convert_to_dtype():
+    from bed_reader._open_bed import _convert_to_dtype
+
+    input = [
+        [["a", "b", "c"], ["a", "b", "c"], None, None],
+        [["1.0", "2.0", "3.0"], ["1.0", "2.0", "3.0"], [1, 2, 3], [1.0, 2.0, 3.0]],
+        [["1.0", "2.0", "3.5"], ["1.0", "2.0", "3.5"], None, [1.0, 2.0, 3.5]],
+        [["1", "2", "3"], ["1", "2", "3"], [1, 2, 3], [1.0, 2.0, 3.0]],
+        [["1", "A", "3"], ["1", "A", "3"], None, None],
+    ]
+    # convert all to np.array
+    input = [
+        [np.array(inner) if inner is not None else None for inner in outer]
+        for outer in input
+    ]
+
+    for ori, exp_str, exp_int, exp_float in input:
+        for dtype, exp in (
+            [np.str_, exp_str],
+            [np.int32, exp_int],
+            [
+                np.float32,
+                exp_float,
+            ],
+        ):
+            try:
+                actual = _convert_to_dtype(ori, dtype)
+                assert np.array_equal(actual, exp)
+            except ValueError as e:
+                print(e)
+                assert exp is None
+
+
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
 

bed_reader-1.0.0b4/bed_reader/tests/test_opt_dep.py

@@ -0,0 +1,29 @@
+import pytest
+
+from bed_reader import open_bed, sample_file
+
+
+def test_optional_dependencies(shared_datadir):
+    try:
+        import scipy.sparse as sprase
+    except ImportError:
+        sprase = None
+
+    file = shared_datadir / "plink_sim_10s_100v_10pmiss.bed"
+    with open_bed(file) as bed:
+        if sprase is None:
+            with pytest.raises(ImportError):
+                _ = bed.read_sparse(dtype="int8")
+        else:
+            _ = bed.read_sparse(dtype="int8")
+
+    try:
+        import pooch
+    except ImportError:
+        pooch = None
+
+    if pooch is None:
+        with pytest.raises(ImportError):
+            _ = sample_file("small.bed")
+    else:
+        _ = sample_file("small.bed")

bed_reader-1.0.0b4/ignore.py

@@ -0,0 +1,5 @@
+from bed_reader import open_bed, sample_file
+
+file_name = sample_file("sparse.bed")
+with open_bed(file_name) as bed:
+    val_sparse = bed.read_sparse(dtype="int8")

{bed_reader-1.0.0b1 → bed_reader-1.0.0b4}/pyproject.toml

@@ -10,7 +10,7 @@ classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python",
 ]
-dependencies = ["numpy>=1.13.3", "pandas>=0.25.1"]
+dependencies = ["numpy>=1.13.3"]
 
 [project.optional-dependencies]
 samples = ["pooch>=1.5.0"]

bed_reader-1.0.0b4/requirements-min-dev.txt

@@ -0,0 +1,3 @@
+numpy>=1.13.3
+pytest
+pytest-datadir

{bed_reader-1.0.0b1 → bed_reader-1.0.0b4}/requirements.txt

@@ -1,4 +1,3 @@
 numpy>=1.13.3
-pandas>=0.25.1
 scipy>=1.4
 pooch>=1.5.0

{bed_reader-1.0.0b1 → bed_reader-1.0.0b4}/useful.bat

@@ -11,4 +11,7 @@ maturin develop --target-dir c:\deldir\
 pytest --doctest-modules bed_reader\_open_bed.py
 
 # generate doc
-\doc>make html & build\html\index.html
+\doc>make html & build\html\index.html
+
+# create a local *.whl
+maturin build --release

bed_reader-1.0.0b1/requirements-minimal.txt

@@ -1 +0,0 @@
-numpy>=1.13.3