cap-anndata 0.3.1__tar.gz → 0.4.0__tar.gz

{cap_anndata-0.3.1 → cap_anndata-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: cap_anndata
-Version: 0.3.1
+Version: 0.4.0
 Summary: Partial read/write of AnnData (h5ad) files for low-memory operations with large datasets.
 Home-page: https://github.com/cellannotation/cap-anndata
 Author: R. Mukhin, A. Isaev
@@ -20,6 +20,17 @@ Requires-Dist: anndata>=0.10.0
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0.0; extra == "dev"
 Requires-Dist: setuptools~=69.1.1; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # CAP-AnnData: Partial I/O for AnnData (.h5ad) Files
 

cap_anndata-0.4.0/cap_anndata/backed_df.py

@@ -0,0 +1,81 @@
+import pandas as pd
+import numpy as np
+from typing import List, Any, Union
+
+from pandas._typing import Self
+from pandas.core.generic import bool_t
+
+
+class CapAnnDataDF(pd.DataFrame):
+    """
+    The class to expand the pandas DataFrame behaviour to support partial
+    reading and writing of AnnData obs and var (raw.var) fields.
+    The main feature of the class is handling <column-order> attribute
+    which must be a copy of h5py.Group attribute
+    """
+
+    _metadata = ["column_order"]
+
+    def column_order_array(self) -> np.array:
+        order = self.column_order
+        if order is not None and isinstance(order, List):
+            # Convert it to numpy array of str elements
+            return np.array(order, dtype=object)
+        else:
+            return order
+
+    def rename_column(self, old_name: str, new_name: str) -> None:
+        i = np.where(self.column_order_array() == old_name)[0]
+        tmp_array = self.column_order_array().copy()
+        tmp_array[i] = new_name
+        self.column_order = tmp_array.copy()
+        self.rename(columns={old_name: new_name}, inplace=True)
+
+    def remove_column(self, col_name: str) -> None:
+        i = np.where(self.column_order_array() == col_name)[0]
+        self.column_order = np.delete(self.column_order_array(), i)
+        self.drop(columns=[col_name], inplace=True)
+
+    def __setitem__(self, key, value) -> None:
+        if key not in self.column_order_array():
+            self.column_order = np.append(self.column_order_array(), key)
+        return super().__setitem__(key, value)
+
+    @classmethod
+    def from_df(cls, df: pd.DataFrame, column_order: Union[np.array, List[str], None] = None) -> Self:
+        if column_order is None:
+            column_order = df.columns.to_numpy()
+        elif isinstance(column_order, List):
+            column_order = np.array(column_order)
+        new_inst = cls(df)
+        new_inst.column_order = column_order
+        return new_inst
+
+    def join(self, other: Any, **kwargs) -> Self:
+        result = super().join(other=other, **kwargs)
+        if isinstance(other, CapAnnDataDF):
+            new_columns = [
+                col for col in other.column_order_array() if col not in self.column_order_array()
+            ]
+        else:
+            new_columns = [col for col in other.columns if col not in self.column_order_array()]
+        column_order = np.append(self.column_order_array(), new_columns)
+        df = self.from_df(result, column_order=column_order)
+        return df
+
+    def merge(self, right, **kwargs) -> Self:
+        result = super().merge(right=right, **kwargs)
+        if isinstance(right, CapAnnDataDF):
+            new_columns = [
+                col for col in right.column_order_array() if col not in self.column_order_array()
+            ]
+        else:
+            new_columns = [col for col in right.columns if col not in self.column_order_array()]
+        column_order = np.append(self.column_order_array(), new_columns)
+        df = self.from_df(result, column_order=column_order)
+        return df
+
+    def copy(self, deep: Union[bool_t, None] = True) -> Self:
+        column_order = self.column_order_array()
+        df = self.from_df(super().copy(deep=deep), column_order=column_order)
+        return df

{cap_anndata-0.3.1 → cap_anndata-0.4.0}/cap_anndata/cap_anndata.py

@@ -7,19 +7,33 @@ import scipy.sparse as ss
 from packaging import version
 
 if version.parse(ad.__version__) < version.parse("0.11.0"):
-    from anndata.experimental import sparse_dataset, read_elem, write_elem
+    from anndata.experimental import (
+        sparse_dataset,
+        read_elem,
+        write_elem,
+        CSRDataset,
+        CSCDataset,
+    )
 else:
-    from anndata.io import sparse_dataset, read_elem, write_elem
+    from anndata.io import (
+        sparse_dataset,
+        read_elem,
+        write_elem,
+    )
+    from anndata.abc import (
+        CSRDataset,
+        CSCDataset,
+    )
 
 from cap_anndata import CapAnnDataDF, CapAnnDataDict
 
 logger = logging.getLogger(__name__)
 
 X_NOTATION = Union[
-    h5py.Dataset, ad.experimental.CSRDataset, ad.experimental.CSCDataset, None
+    h5py.Dataset, CSRDataset, CSCDataset, None
 ]
 ARRAY_MAPPING_NOTATION = CapAnnDataDict[str, X_NOTATION]
-
+FIELDS_SUPPORTED_TO_OVERWRITE = ["obs", "var", "raw.var", "uns", "layers", "obsm", "varm", "obsp", "varp"]
 NotLinkedObject: Final = "__NotLinkedObject"
 
 
@@ -57,15 +71,7 @@ class BaseLayerMatrixAndDf:
         return shape
 
     def _lazy_df_load(self, key: str) -> CapAnnDataDF:
-        df = CapAnnDataDF()
-        attribute = self._path_to_content + key
-        column_order = self._read_attr(self._file[attribute], "column-order")
-        df.column_order = column_order
-        if df.column_order.dtype != object:
-            # empty DataFrame will have column_order as float64
-            # which leads to failure in overwrite method
-            df.column_order = df.column_order.astype(object)
-        return df
+        return self._read_df(key=key, columns=[])
 
     @staticmethod
     def _read_attr(obj: Union[h5py.Group, h5py.Dataset], attr_name: str) -> any:
@@ -93,8 +99,10 @@ class BaseLayerMatrixAndDf:
             cols_to_read = [c for c in columns if c in column_order]
             df = CapAnnDataDF()
             df.column_order = column_order
+
             index_col = self._read_attr(h5_group, "_index")
-            df.index = read_elem(h5_group[index_col])
+            index = read_elem(h5_group[index_col])
+            df.index = index
 
             for col in cols_to_read:
                 df[col] = read_elem(h5_group[col])
@@ -366,37 +374,43 @@ class CapAnnData(BaseLayerMatrixAndDf):
         return list(self.obsm.keys())
 
     def obs_keys(self) -> List[str]:
-        return self.obs.column_order.tolist()
+        return self.obs.column_order_array().tolist()
 
     def var_keys(self) -> List[str]:
-        return self.var.column_order.tolist()
+        return self.var.column_order_array().tolist()
+    
+    def field_to_entity(self, key):
+        if key == "obs":
+            return self.obs
+        elif key == "var":
+            return self.var
+        elif key == "raw.var":
+            return self.raw.var if self.raw is not None else None
+        elif key == "uns":
+            return self.uns
+        elif key == "layers":
+            return self.layers
+        elif key == "obsm":
+            return self.obsm
+        elif key == "varm":
+            return self.varm
+        elif key == "obsp":
+            return self.obsp
+        elif key == "varp":
+            return self.varp
+        else:
+            raise KeyError(
+                f"The field {key} is not supported! The list of supported fields are equal to {FIELDS_SUPPORTED_TO_OVERWRITE} "
+                f"attributes of the CapAnnData class."
+            )
 
     def overwrite(self, fields: List[str] = None, compression: str = "lzf") -> None:
-        field_to_entity = {
-            "obs": self.obs,
-            "var": self.var,
-            "raw.var": self.raw.var if self.raw is not None else None,
-            "uns": self.uns,
-            "layers": self.layers,
-            "obsm": self.obsm,
-            "varm": self.varm,
-            "obsp": self.obsp,
-            "varp": self.varp,
-        }
-
         if fields is None:
-            fields = list(field_to_entity.keys())
-        else:
-            for f in fields:
-                if f not in field_to_entity.keys():
-                    raise KeyError(
-                        f"The field {f} is not supported! The list of supported fields are equal to supported "
-                        f"attributes of the CapAnnData class: obs, var, raw.var and uns."
-                    )
+            fields = FIELDS_SUPPORTED_TO_OVERWRITE
 
         for key in ["obs", "var", "raw.var"]:
             if key in fields:
-                entity: CapAnnDataDF = field_to_entity[key]
+                entity: CapAnnDataDF = self.field_to_entity(key)
                 if entity is None:
                     continue
 
@@ -407,11 +421,22 @@ class CapAnnData(BaseLayerMatrixAndDf):
                         f"{key}/{col}", entity[col].values, compression=compression
                     )
 
-                column_order = entity.column_order
+                column_order = entity.column_order_array()
                 if (
                     column_order.size == 0
                 ):  # Refs https://github.com/cellannotation/cap-anndata/issues/6
                     column_order = np.array([], dtype=np.float64)
+
+                # Index update
+                index_name = entity.index.name
+                if not index_name:
+                    index_name = "_index"
+                self._file[key].attrs["_index"] = index_name
+                index_col = self._read_attr(self._file[key], "_index")
+                self._write_elem(
+                    f"{key}/{index_col}", entity.index.to_numpy(), compression=compression
+                )
+
                 self._file[key].attrs["column-order"] = column_order
 
         if "uns" in fields:
@@ -424,7 +449,7 @@ class CapAnnData(BaseLayerMatrixAndDf):
 
         for field in ["layers", "obsm", "varm", "obsp", "varp"]:
             if field in fields:
-                for key in field_to_entity[field].keys_to_remove:
+                for key in self.field_to_entity(field).keys_to_remove:
                     del self._file[f"{field}/{key}"]
 
     def create_layer(

{cap_anndata-0.3.1 → cap_anndata-0.4.0}/cap_anndata.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: cap_anndata
-Version: 0.3.1
+Version: 0.4.0
 Summary: Partial read/write of AnnData (h5ad) files for low-memory operations with large datasets.
 Home-page: https://github.com/cellannotation/cap-anndata
 Author: R. Mukhin, A. Isaev
@@ -20,6 +20,17 @@ Requires-Dist: anndata>=0.10.0
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0.0; extra == "dev"
 Requires-Dist: setuptools~=69.1.1; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # CAP-AnnData: Partial I/O for AnnData (.h5ad) Files
 

{cap_anndata-0.3.1 → cap_anndata-0.4.0}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='cap_anndata',
-    version='0.3.1',
+    version='0.4.0',
     author='R. Mukhin, A. Isaev',
     author_email='roman@ebookapplications.com',
     packages=find_packages(exclude=["test"]),

{cap_anndata-0.3.1 → cap_anndata-0.4.0}/test/test_backed_df.py

@@ -41,9 +41,8 @@ def test_remove_column():
 
 def test_from_df_class_method():
     data = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
-    new_df = CapAnnDataDF.from_df(data, ["B", "A"])
-
-    assert list(new_df.column_order) == ["B", "A"]
+    new_df = CapAnnDataDF.from_df(data)
+    assert list(new_df.column_order) == ["A", "B"]
 
 
 def test_column_order_integrity():
@@ -59,23 +58,22 @@ def test_column_order_integrity():
 
 def test_join():
     data1 = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
-    data2 = pd.DataFrame({"D": [7, 8, 9], "E": [10, 11, 12]})
-    cap_anndata_df1 = CapAnnDataDF.from_df(data1, column_order=["A", "B", "C"])
-
-    cap_anndata_df1 = cap_anndata_df1.join(data2, how="left")
+    data2 = pd.DataFrame({"C": [7, 8, 9], "D": [10, 11, 12]})
+    cap_anndata_df1 = CapAnnDataDF.from_df(data1)
+    cap_anndata_df2 = cap_anndata_df1.join(data2, how="left")
 
-    expected_order = ["A", "B", "C", "D", "E"]
-    assert list(cap_anndata_df1.column_order) == expected_order
-    assert cap_anndata_df1.shape == (3, 4)
+    expected_order = ["A", "B", "C", "D"]
+    assert list(cap_anndata_df2.column_order) == expected_order
+    assert cap_anndata_df2.shape == (3, 4)
 
 
 def test_merge():
     data1 = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})
     data2 = pd.DataFrame({"A": [2, 3, 4], "D": [10, 11, 12]})
-    cap_anndata_df1 = CapAnnDataDF.from_df(data1, column_order=["A", "B", "C"])
+    cap_anndata_df1 = CapAnnDataDF.from_df(data1)
 
     cap_anndata_df1 = cap_anndata_df1.merge(data2, how="inner", on="A")
 
-    expected_order = ["A", "B", "C", "D"]
+    expected_order = ["A", "B", "D"]
     assert list(cap_anndata_df1.column_order) == expected_order
     assert cap_anndata_df1.shape == (2, 3)

{cap_anndata-0.3.1 → cap_anndata-0.4.0}/test/test_cap_anndata.py

@@ -103,6 +103,26 @@ def test_partial_read():
     pd.testing.assert_index_equal(adata.raw.var.index, cap_adata.raw.var.index)
 
 
+def test_overwrite_dataframe_before_read_obs():
+    path = "tmp.h5ad"
+    x = np.ones((10, 10), dtype=np.float32)
+    adata = ad.AnnData(X=x)
+    adata.obs["columns"] = "value"
+    adata.write_h5ad(path)
+    del adata
+
+    with read_h5ad(path, True) as adata:
+        # https://github.com/cellannotation/cap-anndata/issues/33
+        adata.obs["new_column"] = "new_value"
+        adata.overwrite(["obs"])
+
+    with read_h5ad(path) as adata:
+        adata.read_obs("new_column")
+        assert (adata.obs["new_column"] == "new_value").all(), "Wrong values in column!"
+
+    os.remove(path)
+
+
 @pytest.mark.parametrize("compression", ["gzip", "lzf"])
 def test_overwrite_df(compression):
     adata = get_filled_anndata()
@@ -110,12 +130,17 @@ def test_overwrite_df(compression):
     file_path = os.path.join(temp_folder, "test_overwrite_df.h5ad")
     adata.write_h5ad(file_path)
 
+    new_obs_index = None
     with read_h5ad(file_path, edit=True) as cap_adata:
+        # Modify 'obs'
         cap_adata.read_obs(columns=["cell_type"])
         cap_adata.obs["cell_type"] = [
             f"new_cell_type_{i%2}" for i in range(cap_adata.shape[0])
         ]
         cap_adata.obs["const_str"] = "some string"
+        # Modify obs 'index'
+        new_obs_index = [s + "_new" for s in cap_adata.obs.index]
+        cap_adata.obs.index = new_obs_index
         ref_obs = cap_adata.obs.copy()
 
         # Modify 'var'
@@ -144,6 +169,7 @@ def test_overwrite_df(compression):
     pd.testing.assert_frame_equal(
         ref_obs, adata.obs[ref_obs.columns.to_list()], check_frame_type=False
     )
+    assert (adata.obs.index == new_obs_index).all(), "Index must be changed!"
 
     # Assert changes in 'var'
     assert all([c in adata.var.columns for c in ref_var.columns])
@@ -689,3 +715,104 @@ def test_modify_obsp_varp(field):
         assert len(getattr(cap_adata, field).keys()) == 0
 
     os.remove(file_path)
+
+
+def test_main_var_layers():
+    var_index = [f"ind_{i}" for i in range(10)]
+    raw_var_index = [f"raw_ind_{i}" for i in range(10)]
+
+    x = np.eye(10, dtype=np.float32)
+    raw_x = x * 2
+    adata = ad.AnnData(X=raw_x)
+    adata.var.index = raw_var_index
+    adata.raw = adata
+    adata.X = x
+    adata.var.index = var_index
+    
+    temp_folder = tempfile.mkdtemp()
+    file_path = os.path.join(temp_folder, "test_main_var_layers.h5ad")
+    adata.write_h5ad(file_path)
+
+    with read_h5ad(file_path) as cap_anndata:
+        assert cap_anndata.var.index.tolist() == var_index
+        assert cap_anndata.raw.var.index.tolist() == raw_var_index
+        assert np.allclose(cap_anndata.X[:], x)
+        assert np.allclose(cap_anndata.raw.X[:], raw_x)
+
+    os.remove(file_path)
+
+
+@pytest.mark.parametrize("name", ["barcodes", "", None])
+def test_modify_index(name):
+    adata = get_base_anndata()
+    
+    temp_folder = tempfile.mkdtemp()
+    file_path = os.path.join(temp_folder, "test_main_var_layers.h5ad")
+    adata.write_h5ad(file_path)
+
+    with read_h5ad(file_path=file_path, edit=True) as cap_adata:
+        cap_adata.read_obs()
+        cap_adata.overwrite(["obs"])
+    
+    cap_adata = ad.read_h5ad(file_path)
+    pd.testing.assert_frame_equal(
+        left=adata.obs,
+        right=cap_adata.obs,
+        check_dtype=True,
+        check_index_type=True,
+        check_names=True,
+    )
+
+    with read_h5ad(file_path=file_path, edit=True) as cap_adata:
+        cap_adata.read_obs()
+        cap_adata.obs.index = pd.Series(data=[f"cell_{i}" for i in range(cap_adata.shape[0])], name=name)
+        cap_adata.overwrite(["obs"])
+    
+    with read_h5ad(file_path=file_path, edit=False) as cap_adata:
+        cap_adata.read_obs()
+        obs = cap_adata.obs
+    
+        assert obs is not None, "DataFrame must be loaded!"
+        assert obs.index is not None, "DataFrame must have Index!"
+        if not name:
+            assert obs.index.name == None, "Index name must not be set!"
+        else:
+            assert obs.index.name == name, "Index name must be set!"
+        assert obs.index.to_list() == [f"cell_{i}" for i in range(cap_adata.shape[0])], "Wrong index values!"
+
+
+def test_column_order_changes():
+    adata = get_base_anndata(n_rows = 3, n_genes = 2, sparse=False)
+
+    temp_folder = tempfile.mkdtemp()
+    file_path = os.path.join(temp_folder, "test_column_order.h5ad")
+    adata.write_h5ad(file_path)
+
+    data = {"A": [1, 2, 3], "B": [4, 5, 6]}
+    with read_h5ad(file_path=file_path, edit=True) as cap_adata:
+        df = pd.DataFrame(data)
+        cap_df = CapAnnDataDF.from_df(df)
+        cap_adata.obs = CapAnnDataDF.from_df(cap_df)
+        cap_adata.overwrite(["obs"])
+
+    new_column_order = list(data.keys())
+    new_column_order.reverse()
+    with read_h5ad(file_path=file_path, edit=True) as cap_adata:
+        cap_adata.read_obs()
+        df = cap_adata.obs[new_column_order] # change order via dataframe
+        cap_df = CapAnnDataDF.from_df(df)
+        cap_adata.obs = cap_df
+        cap_adata.overwrite(["obs"])
+
+    new_column_order.reverse()
+    with read_h5ad(file_path=file_path, edit=True) as cap_adata:
+        cap_adata.read_obs()
+        cap_df = cap_adata.obs
+        cap_df.column_order = new_column_order # change order via column_order
+        cap_adata.obs = cap_df
+        cap_adata.overwrite(["obs"])
+
+    with read_h5ad(file_path=file_path) as cap_adata:
+        cap_adata.read_obs()
+        assert list(cap_adata.obs.column_order) == new_column_order
+        assert list(cap_adata.obs.columns) == new_column_order

cap_anndata-0.3.1/cap_anndata/backed_df.py

@@ -1,69 +0,0 @@
-import pandas as pd
-import numpy as np
-from typing import List, Any, Union
-import logging
-
-from pandas._typing import Self
-from pandas.core.generic import bool_t
-
-logger = logging.getLogger(__name__)
-
-
-class CapAnnDataDF(pd.DataFrame):
-    """
-    The class to expand the pandas DataFrame behaviour to support partial
-    reading and writing of AnnData obs and var (raw.var) fields.
-    The main feature of the class is handling <column-order> attribute
-    which must be a copy of h5py.Group attribute
-    """
-
-    _metadata = ["column_order"]
-
-    def rename_column(self, old_name: str, new_name: str) -> None:
-        i = np.where(self.column_order == old_name)[0]
-        self.column_order[i] = new_name
-        self.rename(columns={old_name: new_name}, inplace=True)
-
-    def remove_column(self, col_name: str) -> None:
-        i = np.where(self.column_order == col_name)[0]
-        self.column_order = np.delete(self.column_order, i)
-        self.drop(columns=[col_name], inplace=True)
-
-    def __setitem__(self, key, value) -> None:
-        if key not in self.column_order:
-            self.column_order = np.append(self.column_order, key)
-        return super().__setitem__(key, value)
-
-    @classmethod
-    def from_df(cls, df: pd.DataFrame, column_order: List[str] = None) -> Self:
-        if column_order is None:
-            column_order = df.columns.to_numpy()
-
-        new_inst = cls(df)
-        new_inst.column_order = column_order
-        return new_inst
-
-    def join(self, other: Any, **kwargs) -> Self:
-        result = super().join(other=other, **kwargs)
-        if isinstance(other, CapAnnDataDF):
-            new_columns = [
-                col for col in other.column_order if col not in self.column_order
-            ]
-        else:
-            new_columns = [col for col in other.columns if col not in self.column_order]
-        column_order = np.append(self.column_order, new_columns)
-        return self.from_df(result, column_order=column_order)
-
-    def merge(self, right, **kwargs) -> Self:
-        result = super().merge(right=right, **kwargs)
-        if isinstance(right, CapAnnDataDF):
-            new_columns = [
-                col for col in right.column_order if col not in self.column_order
-            ]
-        else:
-            new_columns = [col for col in right.columns if col not in self.column_order]
-        column_order = np.append(self.column_order, new_columns)
-        return self.from_df(result, column_order=column_order)
-
-    def copy(self, deep: Union[bool_t, None] = True) -> Self:
-        return self.from_df(super().copy(deep=deep), column_order=self.column_order)