bed-reader 1.0.2__tar.gz → 1.0.3b1__tar.gz

{bed_reader-1.0.2 → bed_reader-1.0.3b1}/CHANGELOG.md

@@ -5,7 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [1.0.1] - 2024-4-16
+## [1.0.2] - 2024-4-17
+
+- On the Python file, add `create-bed`, a method for writing large
+  bed files SNP-by-SNP (or individual-by-individual).
+  Also, add full support for the less common individual-major files.
+
+## [1.0.1] - 2024-3-16
 
 - Add support for cloud files to both Rust and Python.
 

{bed_reader-1.0.2 → bed_reader-1.0.3b1}/Cargo.lock

@@ -136,7 +136,7 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
 [[package]]
 name = "bed-reader"
-version = "1.0.2"
+version = "1.0.3-beta.1"
 dependencies = [
  "anyhow",
  "anyinput",

{bed_reader-1.0.2 → bed_reader-1.0.3b1}/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-version = "1.0.2"
+version = "1.0.3-beta.1"
 name = "bed-reader"
 description = "Read and write the PLINK BED format, simply and efficiently."
 repository = "https://github.com/fastlmm/bed-reader"

{bed_reader-1.0.2 → bed_reader-1.0.3b1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: bed-reader
-Version: 1.0.2
+Version: 1.0.3b1
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10

{bed_reader-1.0.2 → bed_reader-1.0.3b1}/bed_reader/__init__.py

@@ -7,9 +7,10 @@
 
 from bed_reader._open_bed import get_num_threads, open_bed
 from bed_reader._sample_data import sample_file, sample_url, tmp_path
-from bed_reader._to_bed import to_bed
+from bed_reader._to_bed import create_bed, to_bed
 
 from .bed_reader import (
+    encode1_i8,
     file_aat_piece_f32_orderf,
     file_aat_piece_f64_orderf,
     file_ata_piece_f32_orderf,

{bed_reader-1.0.2 → bed_reader-1.0.3b1}/bed_reader/_open_bed.py

@@ -301,7 +301,7 @@ class open_bed:
                 check_file_cloud(self.location.geturl(), self.cloud_options)
             else:
                 with open(self.location, "rb") as filepointer:
-                    self._check_file(filepointer)
+                    self._mode = self._check_file(filepointer)
 
     # # its an error to set both location and filepath
     # location = self._combined(location, filepath, "location", "filepath")
@@ -547,6 +547,7 @@ class open_bed:
                     )
 
         else:
+
             if not self.count_A1:
                 byteZero = 0
                 byteThree = 2
@@ -563,45 +564,77 @@ class open_bed:
             # Also, note that reading with python will often result in
             # non-contiguous memory
             # logging.warn("using pure python plink parser (might be much slower!!)")
+
+            if self.major == "SNP":
+                minor_count = self.iid_count
+                minor_index = iid_index
+                major_index = sid_index
+            else:
+                minor_count = self.sid_count
+                minor_index = sid_index
+                major_index = iid_index
+
             val = np.zeros(
-                ((int(np.ceil(0.25 * self.iid_count)) * 4), len(sid_index)),
+                ((int(np.ceil(0.25 * minor_count)) * 4), len(major_index)),
                 order=order,
                 dtype=dtype,
             )  # allocate it a little big
 
-            nbyte = int(np.ceil(0.25 * self.iid_count))
+            nbyte = int(np.ceil(0.25 * minor_count))
             with open(self.location, "rb") as filepointer:
-                for SNPsIndex, bimIndex in enumerate(sid_index):
-                    startbit = int(np.ceil(0.25 * self.iid_count) * bimIndex + 3)
+                for major_index_value, major_index_index in enumerate(major_index):
+                    startbit = int(np.ceil(0.25 * minor_count) * major_index_index + 3)
                     filepointer.seek(startbit)
                     bytes = np.array(bytearray(filepointer.read(nbyte))).reshape(
-                        (int(np.ceil(0.25 * self.iid_count)), 1), order="F"
+                        (int(np.ceil(0.25 * minor_count)), 1), order="F"
                     )
 
-                    val[3::4, SNPsIndex : SNPsIndex + 1] = byteZero
-                    val[3::4, SNPsIndex : SNPsIndex + 1][bytes >= 64] = missing
-                    val[3::4, SNPsIndex : SNPsIndex + 1][bytes >= 128] = 1
-                    val[3::4, SNPsIndex : SNPsIndex + 1][bytes >= 192] = byteThree
+                    val[3::4, major_index_value : major_index_value + 1] = byteZero
+                    val[3::4, major_index_value : major_index_value + 1][
+                        bytes >= 64
+                    ] = missing
+                    val[3::4, major_index_value : major_index_value + 1][
+                        bytes >= 128
+                    ] = 1
+                    val[3::4, major_index_value : major_index_value + 1][
+                        bytes >= 192
+                    ] = byteThree
                     bytes = np.mod(bytes, 64)
-                    val[2::4, SNPsIndex : SNPsIndex + 1] = byteZero
-                    val[2::4, SNPsIndex : SNPsIndex + 1][bytes >= 16] = missing
-                    val[2::4, SNPsIndex : SNPsIndex + 1][bytes >= 32] = 1
-                    val[2::4, SNPsIndex : SNPsIndex + 1][bytes >= 48] = byteThree
+                    val[2::4, major_index_value : major_index_value + 1] = byteZero
+                    val[2::4, major_index_value : major_index_value + 1][
+                        bytes >= 16
+                    ] = missing
+                    val[2::4, major_index_value : major_index_value + 1][
+                        bytes >= 32
+                    ] = 1
+                    val[2::4, major_index_value : major_index_value + 1][
+                        bytes >= 48
+                    ] = byteThree
                     bytes = np.mod(bytes, 16)
-                    val[1::4, SNPsIndex : SNPsIndex + 1] = byteZero
-                    val[1::4, SNPsIndex : SNPsIndex + 1][bytes >= 4] = missing
-                    val[1::4, SNPsIndex : SNPsIndex + 1][bytes >= 8] = 1
-                    val[1::4, SNPsIndex : SNPsIndex + 1][bytes >= 12] = byteThree
+                    val[1::4, major_index_value : major_index_value + 1] = byteZero
+                    val[1::4, major_index_value : major_index_value + 1][
+                        bytes >= 4
+                    ] = missing
+                    val[1::4, major_index_value : major_index_value + 1][bytes >= 8] = 1
+                    val[1::4, major_index_value : major_index_value + 1][
+                        bytes >= 12
+                    ] = byteThree
                     bytes = np.mod(bytes, 4)
-                    val[0::4, SNPsIndex : SNPsIndex + 1] = byteZero
-                    val[0::4, SNPsIndex : SNPsIndex + 1][bytes >= 1] = missing
-                    val[0::4, SNPsIndex : SNPsIndex + 1][bytes >= 2] = 1
-                    val[0::4, SNPsIndex : SNPsIndex + 1][bytes >= 3] = byteThree
-                val = val[iid_index, :]  # reorder or trim any extra allocation
+                    val[0::4, major_index_value : major_index_value + 1] = byteZero
+                    val[0::4, major_index_value : major_index_value + 1][
+                        bytes >= 1
+                    ] = missing
+                    val[0::4, major_index_value : major_index_value + 1][bytes >= 2] = 1
+                    val[0::4, major_index_value : major_index_value + 1][
+                        bytes >= 3
+                    ] = byteThree
+                val = val[minor_index, :]  # reorder or trim any extra allocation
                 assert val.dtype == np.dtype(dtype)  # real assert
                 if not open_bed._array_properties_are_ok(val, order):
                     val = val.copy(order=order)
-
+            # if in force python mode, and individual-major mode, then we need to transpose
+            if self.major == "individual":
+                val = val.T
         return val
 
     def _pick_reader(self, dtype):
@@ -633,6 +666,47 @@ class open_bed:
     def __str__(self) -> str:
         return f"{self.__class__.__name__}('{self.location}',...)"
 
+    @property
+    def major(self) -> str:
+        """
+        Major mode of a local .bed file.
+
+        Returns
+        -------
+        str
+            'SNP' or 'individual'
+
+
+        Almost all PLINK 1.9 .bed files are 'SNP' major. This makes
+        reading the data by SNP(s) fast.
+
+        Errors
+        ------
+        ValueError
+            If the file is a cloud file.
+
+        Example
+        -------
+
+        .. doctest::
+
+            >>> from bed_reader import open_bed, sample_file
+            >>>
+            >>> file_name = sample_file("small.bed")
+            >>> with open_bed(file_name) as bed:
+            ...     print(bed.major)
+            SNP
+
+        """
+        if self._is_url(self.location):
+            raise ValueError("Cannot determine major mode for cloud files")
+        # if self._mode is not set, set it
+        if not hasattr(self, "mode"):
+            with open(self.location, "rb") as filepointer:
+                self._mode = self._check_file(filepointer)
+
+        return "individual" if self._mode == b"\x00" else "SNP"
+
     @property
     def fid(self) -> np.ndarray:
         """
@@ -1158,12 +1232,14 @@ class open_bed:
 
     @staticmethod
     def _check_file(filepointer):
-        mode = filepointer.read(2)
-        if mode != b"l\x1b":
+        magic_number = filepointer.read(2)
+        if magic_number != b"l\x1b":
+            raise ValueError("Not a valid .bed file")
+        mode = filepointer.read(1)
+        # Check if mode is either individual-major or SNP-major
+        if mode not in (b"\x00", b"\x01"):
             raise ValueError("Not a valid .bed file")
-        mode = filepointer.read(1)  # \x01 = SNP major \x00 = individual major
-        if mode != b"\x01":
-            raise ValueError("only SNP-major is implemented")
+        return mode
 
     def __del__(self):
         self.__exit__()
@@ -1389,7 +1465,7 @@ class open_bed:
         num_threads=None,
         max_concurrent_requests=None,
         max_chunk_bytes=None,
-    ) -> (Union[sparse.csc_matrix, sparse.csr_matrix]) if sparse is not None else None:
+    ) -> (Union[sparse.csc_matrix, sparse.csr_matrix]) if sparse is not None else None:  # type: ignore
         """
         Read genotype information into a :mod:`scipy.sparse` matrix. Sparse matrices
         may be useful when the data is mostly zeros.

{bed_reader-1.0.2 → bed_reader-1.0.3b1}/bed_reader/_sample_data.py

@@ -71,7 +71,7 @@ def sample_file(filepath: Union[str, Path]) -> str:
     if pooch is None:
         raise ImportError(
             "The function sample_file() requires pooch. "
-            + "Install it with 'pip install --upgrade bed-reader[sample]'."
+            + "Install it with 'pip install --upgrade bed-reader[samples]'."
         )
 
     filepath = Path(filepath)