lamindb 0.49.3__py3-none-any.whl → 0.50.1__py3-none-any.whl

lamindb/dev/_feature_manager.py

@@ -0,0 +1,102 @@
+from typing import Dict, List, Union
+
+import pandas as pd
+from lnschema_core.models import Dataset, FeatureSet, File
+
+from .._query_set import QuerySet
+
+
+def create_features_df(
+    file: File, feature_sets: List[FeatureSet], exclude: bool = True
+):
+    features = []
+    for feature_set in feature_sets:
+        if exclude:
+            features_df = feature_set.features.exclude(registries__isnull=True).df()
+        else:
+            features_df = feature_set.features.df()
+        slots = file.feature_sets.through.objects.filter(
+            file=file, feature_set=feature_set
+        ).list("slot")
+        for slot in slots:
+            features_df["slot"] = slot
+            features.append(features_df)
+    features_df = pd.concat(features)
+    return features_df.sort_values(["slot", "registries"])
+
+
+def get_accessor_by_orm(host: Union[File, Dataset]) -> Dict:
+    dictionary = {
+        field.related_model.__get_name_with_schema__(): field.name
+        for field in host._meta.related_objects
+    }
+    dictionary["core.Feature"] = "features"
+    dictionary["core.Label"] = "labels"
+    return dictionary
+
+
+def get_feature_set_by_slot(host) -> Dict:
+    # if the host is not yet saved
+    if host._state.adding:
+        return host._feature_sets
+    # otherwise, we need a query
+    feature_set_links = host.feature_sets.through.objects.filter(file_id=host.id)
+    return {
+        feature_set_link.slot: FeatureSet.objects.get(
+            id=feature_set_link.feature_set_id
+        )
+        for feature_set_link in feature_set_links
+    }
+
+
+class FeatureManager:
+    """Feature manager (:attr:`~lamindb.dev.Data.features`).
+
+    See :class:`~lamindb.dev.Data` for more information.
+    """
+
+    def __init__(self, host: Union[File, Dataset]):
+        self._host = host
+        self._feature_set_by_slot = get_feature_set_by_slot(host)
+        self._accessor_by_orm = get_accessor_by_orm(host)
+
+    def __repr__(self) -> str:
+        if len(self._feature_set_by_slot) > 0:
+            msg = ""
+            for slot, feature_set in self._feature_set_by_slot.items():
+                msg += f"'{slot}': {feature_set}\n"
+            return msg
+        else:
+            return "no linked features"
+
+    def __getitem__(self, slot) -> QuerySet:
+        if slot not in self._feature_set_by_slot:
+            raise ValueError(
+                f"No linked feature set for slot: {slot}\nDid you get validation"
+                " warnings? Only features that match registered features get validated"
+                " and linked."
+            )
+        feature_set = self._feature_set_by_slot[slot]
+        orm_name = feature_set.registry
+        return getattr(feature_set, self._accessor_by_orm[orm_name]).all()
+
+    def add_feature_set(self, feature_set: FeatureSet, slot: str):
+        """Add new feature set to a slot.
+
+        Args:
+            feature_set: `FeatureSet` A feature set object.
+            slot: `str` The access slot.
+        """
+        if self._host._state.adding:
+            raise ValueError(
+                "Please save the file or dataset before adding a feature set!"
+            )
+        feature_set.save()
+        link_record = self._host.feature_sets.through.objects.filter(
+            file=self._host, feature_set=feature_set, slot=slot
+        ).one_or_none()
+        if link_record is None:
+            self._host.feature_sets.through(
+                file=self._host, feature_set=feature_set, slot=slot
+            ).save()
+            self._feature_set_by_slot[slot] = feature_set

lamindb/dev/_settings.py

@@ -14,7 +14,7 @@ class Settings:
     """
 
     def __init__(self):
-        self._verbosity: int = 2  # info-level logging
+        self._verbosity: int = 4  # hint-level logging
         logger.set_verbosity(self._verbosity)
 
     upon_file_create_if_hash_exists: Literal[
@@ -34,7 +34,7 @@ class Settings:
     It speeds up file creation by about a factor 100.
     """
     upon_create_search_names: bool = True
-    """To speed up creating ORM objects (default `True`).
+    """To speed up creating Registry objects (default `True`).
 
     If `True`, search for alternative names.
 
@@ -80,13 +80,14 @@ class Settings:
 
     @property
     def verbosity(self) -> int:
-        """Verbosity (default 3).
-
-        - 0: only show 'error' messages
-        - 1: also show 'warning' messages
-        - 2: also show 'info' messages
-        - 3: also show 'hint' messages
-        - 4: also show detailed 'debug' messages
+        """Verbosity (default 4 / 'hint').
+
+        - 0: ❌ only show 'error' messages
+        - 1: 🔶 also show 'warning' messages
+        - 2: ✅ also show 'success' messages
+        - 3: 💬 also show 'info' messages
+        - 4: 💡 also show 'hint' messages
+        - 5: 🐛 also show detailed 'debug' messages
 
         This is based on Scanpy's and Django's verbosity setting.
         """

lamindb/dev/_view_parents.py

@@ -1,7 +1,12 @@
 from typing import List, Set, Union
 
-from lnschema_core import ORM, File, Run
-from lnschema_core.models import format_datetime
+from lnschema_core import File, Registry, Run, Transform
+from lnschema_core.models import format_field_value
+
+LAMIN_GREEN_LIGHTER = "#10b981"
+LAMIN_GREEN_DARKER = "#065f46"
+GREEN_FILL = "honeydew"
+TRANSFORM_EMOJIS = {"notebook": "📔", "app": "🖥️", "pipeline": "🧩"}
 
 
 def view_lineage(file: File, with_children: bool = True):
@@ -33,8 +38,8 @@ def view_lineage(file: File, with_children: bool = True):
     u = graphviz.Digraph(
         file.id,
         node_attr={
-            "fillcolor": "honeydew",
-            "color": "seagreen",
+            "fillcolor": GREEN_FILL,
+            "color": LAMIN_GREEN_DARKER,
             "fontname": "Helvetica",
             "fontsize": "10",
         },
@@ -51,7 +56,7 @@ def view_lineage(file: File, with_children: bool = True):
         else:
             style = "rounded,filled"
             shape = "box"
-            fillcolor = "honeydew"
+            fillcolor = GREEN_FILL
         u.node(
             node_id,
             label=node_label,
@@ -66,12 +71,12 @@ def view_lineage(file: File, with_children: bool = True):
             add_node(row["target_record"], row["target"], row["target_label"], u)
 
         u.edge(row["source"], row["target"], color="dimgrey")
-    # label the searched file mediumseagreen
+    # label the searched file
     u.node(
         file.id,
         label=file_label,
         style="rounded,filled",
-        fillcolor="mediumseagreen",
+        fillcolor=LAMIN_GREEN_LIGHTER,
         shape="box",
     )
 
@@ -79,7 +84,7 @@ def view_lineage(file: File, with_children: bool = True):
 
 
 def view_parents(
-    record: ORM, field: str, with_children: bool = False, distance: int = 100
+    record: Registry, field: str, with_children: bool = False, distance: int = 100
 ):
     """Graph of parents."""
     if not hasattr(record, "parents"):
@@ -105,8 +110,8 @@ def view_parents(
     u = graphviz.Digraph(
         record.id,
         node_attr={
-            "color": "seagreen",
-            "fillcolor": "honeydew",
+            "color": LAMIN_GREEN_DARKER,
+            "fillcolor": GREEN_FILL,
             "shape": "box",
             "style": "rounded,filled",
             "fontname": "Helvetica",
@@ -116,17 +121,17 @@ def view_parents(
     )
     u.node(
         record_label.replace(":", "_"),
-        label=record_label,
-        fillcolor="mediumseagreen",
+        label=_add_emoji(record, record_label),
+        fillcolor=LAMIN_GREEN_LIGHTER,
     )
     for _, row in df_edges.iterrows():
-        u.node(row["source"], label=row["source_label"])
+        u.node(row["source"], label=_add_emoji(record, row["source_label"]))
         u.edge(row["source"], row["target"], color="dimgrey")
 
     return u
 
 
-def _get_parents(record: ORM, field: str, distance: int, children: bool = False):
+def _get_parents(record: Registry, field: str, distance: int, children: bool = False):
     """Recursively get parent records within a distance."""
     if children:
         key = "parents"
@@ -152,7 +157,7 @@ def _get_parents(record: ORM, field: str, distance: int, children: bool = False)
 
 
 def _df_edges_from_parents(
-    record: ORM, field: str, distance: int, children: bool = False
+    record: Registry, field: str, distance: int, children: bool = False
 ):
     """Construct a DataFrame of edges as the input of graphviz.Digraph."""
     key = "children" if children else "parents"
@@ -178,6 +183,16 @@ def _df_edges_from_parents(
     return df_edges
 
 
+def _add_emoji(record: Registry, label: str):
+    if record.__class__.__name__ == "Transform":
+        emoji = TRANSFORM_EMOJIS.get(record.type, "💫")
+    elif record.__class__.__name__ == "Run":
+        emoji = TRANSFORM_EMOJIS.get(record.transform.type, "💫")
+    else:
+        emoji = ""
+    return f"{emoji} {label}"
+
+
 def _get_all_parent_runs(file: File):
     """Get all input file runs recursively."""
     all_runs = {file.run}
@@ -222,9 +237,9 @@ def _label_file_run(record: Union[File, Run]):
     elif isinstance(record, Run):
         name = f'{record.transform.name.replace("&", "&")}'
         return (
-            rf'<{name}<BR/><FONT COLOR="GREY" POINT-SIZE="10"'
+            rf'<{TRANSFORM_EMOJIS.get(str(record.transform.type), "💫")} {name}<BR/><FONT COLOR="GREY" POINT-SIZE="10"'  # noqa
             rf' FACE="Monospace">id={record.id}<BR/>type={record.transform.type},'
-            rf" user={record.created_by.name}<BR/>run_at={format_datetime(record.run_at)}</FONT>>"  # noqa
+            rf" user={record.created_by.name}<BR/>run_at={format_field_value(record.run_at)}</FONT>>"  # noqa
         )
 
 
@@ -248,3 +263,7 @@ def _df_edges_from_runs(all_runs: List[Run]):
     df["source_label"] = df["source_record"].apply(_label_file_run)
     df["target_label"] = df["target_record"].apply(_label_file_run)
     return df
+
+
+def _transform_emoji(transform: Transform):
+    return TRANSFORM_EMOJIS.get(transform.type, "💫")

lamindb/dev/datasets/__init__.py

@@ -1,17 +1,17 @@
-"""Small example datasets.
+"""Test datasets.
 
 .. autosummary::
    :toctree: .
 
    file_fcs
    file_fcs_alpert19
+   file_tsv_rnaseq_nfcore_salmon_merged_gene_counts
    file_jpg_paradisi05
    file_tiff_suo22
    file_fastq
    file_bam
    file_mini_csv
    dir_scrnaseq_cellranger
-   generate_cell_ranger_files
    df_iris
    df_iris_in_meter
    df_iris_in_meter_batch1
@@ -19,6 +19,7 @@
    anndata_mouse_sc_lymph_node
    anndata_human_immune_cells
    anndata_pbmc68k_reduced
+   anndata_file_pbmc68k_test
    anndata_pbmc3k_processed
    anndata_with_obs
    anndata_suo22_Visium10X
@@ -29,6 +30,7 @@
 """
 
 from ._core import (
+    anndata_file_pbmc68k_test,
     anndata_human_immune_cells,
     anndata_mouse_sc_lymph_node,
     anndata_pbmc3k_processed,
@@ -47,7 +49,7 @@ from ._core import (
     file_jpg_paradisi05,
     file_mini_csv,
     file_tiff_suo22,
-    generate_cell_ranger_files,
+    file_tsv_rnaseq_nfcore_salmon_merged_gene_counts,
     mudata_papalexi21_subset,
     schmidt22_crispra_gws_IFNG,
     schmidt22_perturbseq,

lamindb/dev/datasets/_core.py

@@ -39,6 +39,18 @@ def file_jpg_paradisi05() -> Path:
     return Path(filepath)
 
 
+def file_tsv_rnaseq_nfcore_salmon_merged_gene_counts() -> Path:
+    """Gene counts table from nf-core RNA-seq pipeline.
+
+    Output of: https://nf-co.re/rnaseq
+    """
+    filepath, _ = urlretrieve(
+        "https://lamindb-test.s3.amazonaws.com/salmon.merged.gene_counts.tsv",
+        "salmon.merged.gene_counts.tsv",
+    )
+    return Path(filepath)
+
+
 def file_fastq(in_storage_root=False) -> Path:
     """Mini mock fastq file."""
     basedir = Path(".") if not in_storage_root else settings.storage
@@ -80,21 +92,6 @@ def file_tiff_suo22():
     return Path(filepath)
 
 
-def dir_scrnaseq_cellranger(in_storage_root=False) -> Path:
-    """Directory with exemplary scrnaseq cellranger input and output."""
-    filepath, _ = urlretrieve(
-        "https://lamindb-test.s3.amazonaws.com/cellranger_run_001.zip"
-    )
-    from zipfile import ZipFile
-
-    basedir = Path(".") if not in_storage_root else settings.storage
-    with ZipFile(filepath, "r") as zipObj:
-        # Extract all the contents of zip file in current directory
-        zipObj.extractall(path=basedir)
-
-    return basedir / "cellranger_run_001"
-
-
 def anndata_mouse_sc_lymph_node() -> ad.AnnData:
     """Mouse lymph node scRNA-seq dataset from EBI.
 
@@ -135,6 +132,28 @@ def anndata_pbmc68k_reduced() -> ad.AnnData:
     return ad.read(filepath)
 
 
+def anndata_file_pbmc68k_test() -> Path:
+    """Modified from scanpy.datasets.pbmc68k_reduced().
+
+    Additional slots were added for testing purposes. Returns the filepath.
+
+    To reproduce::
+
+        pbmc68k = ln.dev.datasets.anndata_pbmc68k_reduced()
+        pbmc68k_test = pbmc68k[:30, :200].copy()
+        pbmc68k_test.raw = pbmc68k_test[:, :100]
+        pbmc68k_test.obsp["test"] = sparse.eye(pbmc68k_test.shape[0], format="csr")
+        pbmc68k_test.varp["test"] = sparse.eye(pbmc68k_test.shape[1], format="csr")
+        pbmc68k_test.layers["test"] = sparse.csr_matrix(pbmc68k_test.shape)
+        pbmc68k_test.layers["test"][0] = 1.
+        pbmc68k_test.write("pbmc68k_test.h5ad")
+    """
+    filepath, _ = urlretrieve(
+        "https://lamindb-test.s3.amazonaws.com/pbmc68k_test.h5ad", "pbmc68k_test.h5ad"
+    )
+    return Path(filepath)
+
+
 def anndata_pbmc3k_processed() -> ad.AnnData:
     """Modified from scanpy.pbmc3k_processed()."""
     filepath, _ = urlretrieve(
@@ -272,7 +291,7 @@ def df_iris_in_meter_batch2() -> pd.DataFrame:
     return df_iris.iloc[len(df_iris) // 2 :]
 
 
-def generate_cell_ranger_files(
+def dir_scrnaseq_cellranger(
     sample_name: str, basedir: Union[str, Path] = "./", output_only: bool = True
 ):
     """Generate mock cell ranger outputs.
@@ -281,7 +300,6 @@ def generate_cell_ranger_files(
         sample_name: name of the sample
         basedir: run directory
         output_only: only generate output files
-
     """
     basedir = Path(basedir)
 

lamindb/dev/exc.py

@@ -0,0 +1,4 @@
+class ValidationError(Exception):
+    """Validation error: not mapped in registry."""
+
+    pass

lamindb/dev/storage/_backed_access.py

@@ -94,12 +94,23 @@ def read_dataframe(elem: Union[h5py.Dataset, h5py.Group]):
 def safer_read_partial(elem, indices):
     if get_spec(elem).encoding_type == "":
         if isinstance(elem, h5py.Dataset):
-            return elem[indices]
-        else:
-            raise ValueError(
-                "Can not get a subset of the element of type"
-                f" {type(elem).__name__} with an empty spec."
-            )
+            dims = len(elem.shape)
+            if dims == 2:
+                return elem[indices]
+            elif dims == 1:
+                if indices[0] == slice(None):
+                    return elem[indices[1]]
+                elif indices[1] == slice(None):
+                    return elem[indices[0]]
+        elif isinstance(elem, h5py.Group):
+            try:
+                return SparseDataset(elem)[indices]
+            except Exception:
+                pass
+        raise ValueError(
+            "Can not get a subset of the element of type"
+            f" {type(elem).__name__} with an empty spec."
+        )
     else:
         return read_elem_partial(elem, indices=indices)
 
@@ -140,6 +151,16 @@ if ZARR_INSTALLED:
     GroupTypes.append(zarr.Group)
     StorageTypes.append(zarr.Group)
 
+    def _subset_sparse_zarr(elem: zarr.Group, indices):
+        ds = SparseDataset(elem)
+        has_arrays = isinstance(indices[0], np.ndarray) or isinstance(
+            indices[1], np.ndarray
+        )
+        if not has_arrays and indices == (slice(None), slice(None)):
+            return ds.to_memory()
+        else:
+            return ds[indices]
+
     @registry.register_open("zarr")
     def open(filepath: Union[UPath, Path, str]):  # noqa
         fs, file_path_str = infer_filesystem(filepath)
@@ -156,16 +177,31 @@ if ZARR_INSTALLED:
 
     @registry.register("zarr")
     def safer_read_partial(elem, indices):  # noqa
-        if get_spec(elem).encoding_type == "":
+        encoding_type = get_spec(elem).encoding_type
+        if encoding_type == "":
             if isinstance(elem, zarr.Array):
-                return elem.oindex[indices]
-            else:
-                raise ValueError(
-                    "Can not get a subset of the element of type"
-                    f" {type(elem).__name__} with an empty spec."
-                )
+                dims = len(elem.shape)
+                if dims == 2:
+                    return elem.oindex[indices]
+                elif dims == 1:
+                    if indices[0] == slice(None):
+                        return elem.oindex[indices[1]]
+                    elif indices[1] == slice(None):
+                        return elem.oindex[indices[0]]
+            elif isinstance(elem, zarr.Group):
+                try:
+                    return _subset_sparse_zarr(elem, indices)
+                except Exception:
+                    pass
+            raise ValueError(
+                "Can not get a subset of the element of type"
+                f" {type(elem).__name__} with an empty spec."
+            )
         else:
-            return read_elem_partial(elem, indices=indices)
+            if encoding_type in ("csr_matrix", "csc_matrix"):
+                return _subset_sparse_zarr(elem, indices)
+            else:
+                return read_elem_partial(elem, indices=indices)
 
     # this is needed because accessing zarr.Group.keys() directly is very slow
     @registry.register("zarr")
@@ -213,8 +249,6 @@ ArrayTypes = tuple(ArrayTypes)  # type: ignore
 GroupTypes = tuple(GroupTypes)  # type: ignore
 StorageTypes = tuple(StorageTypes)  # type: ignore
 
-ArrayOrSparseTypes = ArrayTypes + (SparseDataset,)  # type: ignore
-
 
 ArrayType = Union[ArrayTypes]  # type: ignore
 GroupType = Union[GroupTypes]  # type: ignore
@@ -222,8 +256,10 @@ StorageType = Union[StorageTypes]  # type: ignore
 
 
 def _to_memory(elem):
-    if isinstance(elem, ArrayOrSparseTypes):
+    if isinstance(elem, ArrayTypes):
         return elem[()]
+    elif isinstance(elem, SparseDataset):
+        return elem.to_memory()
     else:
         return elem
 

lamindb/dev/storage/file.py

@@ -23,6 +23,43 @@ except ImportError:
 AUTO_KEY_PREFIX = ".lamindb/"
 
 
+KNOWN_SUFFIXES = {
+    # without readers
+    ".txt",
+    ".tsv",
+    ".pdf",
+    ".fastq",
+    ".tar",
+    ".zip",
+    # with readers (see below)
+    ".h5ad",
+    ".parquet",
+    ".csv",
+    ".fcs",
+    ".zarr",
+    ".zrad",
+}
+
+
+def extract_suffix_from_path(path: Union[UPath, Path]) -> str:
+    # this if-clause is based on https://stackoverflow.com/questions/31890341
+    # the rest conscsiously deviates
+    if len(path.suffixes) <= 2:
+        return "".join(path.suffixes)
+    else:
+        msg = "file has more than two suffixes (path.suffixes), "
+        # first check the 2nd-to-last suffix because it might be followed by .gz
+        # or another compression-related suffix
+        if path.suffixes[-2] in KNOWN_SUFFIXES:
+            suffix = "".join(path.suffixes[-2:])
+            msg += f"inferring:'{suffix}'"
+        else:
+            suffix = path.suffixes[-1]
+            msg += f"using only last suffix: '{suffix}'"
+        logger.warning(msg)
+        return suffix
+
+
 # add type annotations back asap when re-organizing the module
 def auto_storage_key_from_file(file: File):
     if file.key is None:
@@ -35,25 +72,11 @@ def attempt_accessing_path(file: File, storage_key: str):
     if file.storage_id == settings.storage.id:
         path = settings.storage.key_to_filepath(storage_key)
     else:
-        logger.warning(
-            "file.path() is slightly slower for files outside default storage"
-        )
+        logger.warning("file.path is slightly slower for files outside default storage")
         storage = Storage.filter(id=file.storage_id).one()
         # find a better way than passing None to instance_settings in the future!
         storage_settings = StorageSettings(storage.root)
         path = storage_settings.key_to_filepath(storage_key)
-    # the following is for backward compat
-    if storage_key.startswith(AUTO_KEY_PREFIX) and not path.exists():
-        logger.warning(
-            "You have auto-keyed files in your storage root, please move them into"
-            f" {AUTO_KEY_PREFIX} within your storage location"
-        )
-        # try legacy_storage_key in root
-        for previous_prefix in ["", "lndb/"]:
-            legacy_storage_key = storage_key.replace(AUTO_KEY_PREFIX, previous_prefix)
-            path = settings.storage.key_to_filepath(legacy_storage_key)
-            if path.exists():
-                return path
     return path
 
 
@@ -168,6 +191,11 @@ def read_fcs(*args, **kwargs):
     return readfcs.read(*args, **kwargs)
 
 
+def read_tsv(path: Union[str, Path]) -> pd.DataFrame:
+    path_sanitized = Path(path)
+    return pd.read_csv(path_sanitized, sep="\t")
+
+
 def load_to_memory(filepath: Union[str, Path, UPath], stream: bool = False):
     """Load a file into memory.
 
@@ -189,6 +217,7 @@ def load_to_memory(filepath: Union[str, Path, UPath], stream: bool = False):
 
     READER_FUNCS = {
         ".csv": pd.read_csv,
+        ".tsv": read_tsv,
         ".h5ad": read_adata_h5ad,
         ".parquet": pd.read_parquet,
         ".fcs": read_fcs,