lamindb 0.48a3__py3-none-any.whl → 0.48.1__py3-none-any.whl

lamindb/__init__.py

@@ -4,11 +4,12 @@ Import the package::
 
    import lamindb as ln
 
-`File` tracks data artifacts in form of files, on-disk (`zarr`, etc.) and
-in-memory data objects (`DataFrame`, `AnnData`, etc.) and allows to link them
-against entities of core schema & custom schemas.
+.. note::
 
-The core schema entities are central to lamindb's API:
+    `File` abstracts over objects in storage from blob-like files (pdf, txt, etc.)
+    to streamable storage backends (HDF5, DuckDB, zarr, TileDB, etc.).
+
+    `Dataset` abstracts over `File` and tables in classical warehouses (BigQuery, Snowflake).
 
 .. autosummary::
    :toctree: .
@@ -16,18 +17,13 @@ The core schema entities are central to lamindb's API:
    File
    Dataset
    Transform
-   Run
-   Feature
    Label
+   Feature
+   FeatureSet
+   Modality
    User
    Storage
-
-More control over feature management:
-
-.. autosummary::
-   :toctree: .
-
-   FeatureSet
+   Run
 
 Functional tools:
 
@@ -37,7 +33,6 @@ Functional tools:
    track
    view
    save
-   delete
 
 Static classes & modules:
 
@@ -45,7 +40,6 @@ Static classes & modules:
    :toctree: .
 
    settings
-   context
    types
    setup
    schema
@@ -53,7 +47,7 @@ Static classes & modules:
 
 """
 
-__version__ = "0.48a3"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.48.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 
@@ -93,6 +87,7 @@ if _INSTANCE_SETUP:
         FeatureSet,
         File,
         Label,
+        Modality,
         Run,
         Storage,
         Transform,

lamindb/_context.py

@@ -116,7 +116,7 @@ def get_notebook_name_colab() -> str:
     return name.rstrip(".ipynb")
 
 
-class context:
+class run_context:
     """Global run context."""
 
     instance: Optional[InstanceSettings] = None
@@ -434,3 +434,6 @@ class context:
                     logger.success(f"Updated: {transform}")
 
         cls.transform = transform
+
+
+context = run_context

lamindb/_dataset.py

@@ -67,7 +67,7 @@ def from_files(dataset: Dataset, *, name: str, files: Iterable[File]) -> Dataset
     feature_set_file_links = File.feature_sets.through.objects.filter(
         file_id__in=file_ids
     )
-    feature_set_ids = [link.featureset_id for link in feature_set_file_links]
+    feature_set_ids = [link.feature_set_id for link in feature_set_file_links]
     feature_sets = FeatureSet.select(id__in=feature_set_ids)
     # validate consistency of feature_sets
     # we only allow one feature set per type
@@ -128,13 +128,16 @@ def delete(dataset: Dataset, storage: bool = False):
 def save(dataset: Dataset):
     if dataset.file is not None:
         dataset.file.save()
-    for feature_set in dataset._feature_sets:
+    feature_sets = dataset._feature_sets
+    if isinstance(feature_sets, dict):
+        feature_sets = feature_sets.values()
+    for feature_set in feature_sets:
         feature_set.save()
     super(Dataset, dataset).save()
     if len(dataset._files) > 0:
         dataset.files.set(dataset._files)
     if len(dataset._feature_sets) > 0:
-        dataset.feature_sets.set(dataset._feature_sets)
+        dataset.feature_sets.set(feature_sets)
 
 
 Dataset.__init__ = __init__

lamindb/_feature.py

@@ -1,5 +1,5 @@
 from itertools import islice
-from typing import List
+from typing import List, Optional, Union
 
 import pandas as pd
 from lamin_utils import logger
@@ -32,6 +32,12 @@ def __init__(self, *args, **kwargs):
     # now we proceed with the user-facing constructor
     if len(args) != 0:
         raise ValueError("Only non-keyword args allowed")
+    type: Optional[Union[type, str]] = kwargs.pop("type") if "type" in kwargs else None
+    if type is not None:
+        type_str = type.__name__ if not isinstance(type, str) else type
+    else:
+        type_str = None
+    kwargs["type"] = type_str
     super(Feature, self).__init__(*args, **kwargs)
 
 
@@ -50,7 +56,7 @@ def from_df(cls, df: "pd.DataFrame") -> List["Feature"]:
     categoricals_with_unmapped_categories = {}
     for name, col in df.items():
         if name in categoricals:
-            types[name] = "categorical"
+            types[name] = "category"
             categorical = categoricals[name]
             if hasattr(
                 categorical, "cat"
@@ -66,7 +72,7 @@ def from_df(cls, df: "pd.DataFrame") -> List["Feature"]:
     features = Feature.from_values(df.columns, field=Feature.name, types=types)
     assert len(features) == len(df.columns)
 
-    if len(categoricals) > 0:
+    if len(categoricals_with_unmapped_categories) > 0:
         n_max = 20
         categoricals_with_unmapped_categories_formatted = "\n      ".join(
             [

lamindb/_feature_manager.py

@@ -0,0 +1,176 @@
+from collections import defaultdict
+from typing import Iterable, List, Optional, Union
+
+import pandas as pd
+from lamin_utils import logger
+from lnschema_core.models import ORM, Dataset, Feature, FeatureSet, File
+
+from ._queryset import QuerySet
+from ._save import save
+
+
+def validate_and_cast_feature(feature) -> Feature:
+    if isinstance(feature, str):
+        feature_name = feature
+        feature = Feature.select(name=feature_name).one_or_none()
+        if feature is None:
+            raise ValueError(
+                f"Please create feature: ln.Feature(name='{feature_name}',"
+                " type='category').save()"
+            )
+    return feature
+
+
+def create_features_df(
+    file: File, feature_sets: List[FeatureSet], exclude: bool = True
+):
+    features = []
+    for feature_set in feature_sets:
+        if exclude:
+            features_df = feature_set.features.exclude(labels_orm__isnull=True).df()
+        else:
+            features_df = feature_set.features.df()
+        slots = file.feature_sets.through.objects.filter(
+            file=file, feature_set=feature_set
+        ).list("slot")
+        for slot in slots:
+            features_df["slot"] = slot
+            features.append(features_df)
+    features_df = pd.concat(features)
+    return features_df.sort_values(["labels_schema", "labels_orm"])
+
+
+class FeatureManager:
+    """Feature manager."""
+
+    def __init__(self, host: Union[File, Dataset]):
+        self._host = host
+        slot_feature_sets = (
+            self._feature_set_df_with_slots().reset_index().set_index("slot")["id"]
+        )
+        self._slots = {
+            slot: self._host.feature_sets.get(id=i)
+            for slot, i in slot_feature_sets.items()
+        }
+
+    def __repr__(self) -> str:
+        if len(self._slots) > 0:
+            msg = "slots:\n"
+            for slot, feature_set in self._slots.items():
+                msg += f"    {slot}: {feature_set}\n"
+            return msg
+        else:
+            return "No linked features."
+
+    def __getitem__(self, slot) -> QuerySet:
+        id = (
+            self._host.feature_sets.through.objects.filter(
+                file_id=self._host.id, slot=slot
+            )
+            .one()
+            .feature_set_id
+        )
+        accessor_by_orm = {
+            field.related_model.__name__: field.name
+            for field in self._host._meta.related_objects
+        }
+        accessor_by_orm["Feature"] = "features"
+        feature_set = self._host.feature_sets.filter(id=id).one()
+        return getattr(feature_set, accessor_by_orm[feature_set.ref_orm]).all()
+
+    def _feature_set_df_with_slots(self) -> pd.DataFrame:
+        """Return DataFrame."""
+        df = self._host.feature_sets.df()
+        df.insert(
+            0,
+            "slot",
+            self._host.feature_sets.through.objects.filter(file_id=self._host.id)
+            .df()
+            .set_index("feature_set_id")
+            .slot,
+        )
+        return df
+
+    def add_labels(
+        self, records: Union[ORM, List[ORM]], feature: Optional[Union[str, ORM]] = None
+    ):
+        """Add one or several labels and associate them with a feature."""
+        if isinstance(records, str) or not isinstance(records, Iterable):
+            records = [records]
+        if isinstance(records[0], str):  # type: ignore
+            raise ValueError(
+                "Please pass a record (an ORM object), not a string, e.g., via: label"
+                f" = ln.Label(name='{records[0]}')"  # type: ignore
+            )
+        if self._host._state.adding:
+            raise ValueError("Please save the file or dataset before adding a label!")
+        feature = validate_and_cast_feature(feature)
+        records_by_orm = defaultdict(list)
+        records_by_feature_orm = defaultdict(list)
+        for record in records:
+            records_by_orm[record.__class__.__name__].append(record)
+            if feature is None:
+                try:
+                    record_feature = (
+                        record._feature
+                        if hasattr(record, "_feature")
+                        else record.feature
+                    )
+                except ValueError:
+                    raise ValueError("Pass feature argument")
+            else:
+                record_feature = feature
+            records_by_feature_orm[(record_feature, record.__class__.__name__)].append(
+                record
+            )
+        schema_and_accessor_by_orm = {
+            field.related_model.__name__: (
+                field.related_model.__get_schema_name__(),
+                field.name,
+            )
+            for field in self._host._meta.related_objects
+        }
+        schema_and_accessor_by_orm["Label"] = ("core", "labels")
+        for orm_name, records in records_by_orm.items():
+            save(records)
+            getattr(self._host, schema_and_accessor_by_orm[orm_name][1]).set(records)
+        accessor_by_orm = {
+            field.related_model.__name__: field.name
+            for field in self._host._meta.related_objects
+        }
+        accessor_by_orm["Feature"] = "features"
+        feature_sets = self._host.feature_sets.all()
+        feature_sets_by_orm = {
+            feature_set.ref_orm: feature_set for feature_set in feature_sets
+        }
+        for (feature, orm_name), records in records_by_feature_orm.items():
+            feature = validate_and_cast_feature(feature)
+            logger.info(f"Linking feature {feature.name} to {orm_name}")
+            feature.labels_orm = orm_name
+            feature.labels_schema = schema_and_accessor_by_orm[orm_name][0]
+            feature.save()
+            # check whether we have to update the feature set that manages labels
+            # (Feature) to account for a new feature
+            feature_set = feature_sets_by_orm["Feature"]
+            accessor = "features"
+            linked_features = getattr(feature_set, accessor)
+            if feature not in linked_features.all():
+                logger.info(
+                    f"Linking feature {feature.name} to feature set {feature_set}"
+                )
+                linked_features.add(feature)
+                feature_set.n += 1
+                feature_set.save()
+
+    def add_feature_set(self, feature_set: FeatureSet, slot: str):
+        if self._host._state.adding:
+            raise ValueError(
+                "Please save the file or dataset before adding a feature set!"
+            )
+        feature_set.save()
+        self._host.feature_sets.add(feature_set)
+        link_record = self._host.feature_sets.through.objects.filter(
+            file=self._host, feature_set=feature_set
+        ).one()
+        link_record.slot = slot
+        link_record.save()

lamindb/_feature_set.py

@@ -4,7 +4,7 @@ import pandas as pd
 from django.db.models.query_utils import DeferredAttribute as Field
 from lamin_utils import logger
 from lamindb_setup.dev._docs import doc_args
-from lnschema_core import ORM, Feature, FeatureSet
+from lnschema_core import ORM, Feature, FeatureSet, ids
 from lnschema_core.types import ListLike
 
 from lamindb.dev.hashing import hash_set
@@ -60,9 +60,14 @@ def __init__(self, *args, **kwargs):
         kwargs.pop("ref_field") if "ref_field" in kwargs else "id"
     )
     type: Optional[Union[type, str]] = kwargs.pop("type") if "type" in kwargs else None
-    readout: Optional[str] = kwargs.pop("readout") if "readout" in kwargs else None
+    modality: Optional[str] = kwargs.pop("modality") if "modality" in kwargs else None
     name: Optional[str] = kwargs.pop("name") if "name" in kwargs else None
-    id: Optional[str] = kwargs.pop("id") if "id" in kwargs else None
+    # hash is only internally used
+    hash: Optional[str] = kwargs.pop("hash") if "hash" in kwargs else None
+    if len(kwargs) > 0:
+        raise ValueError(
+            "Only features, ref_field, type, modality, name are valid keyword arguments"
+        )
 
     # now code
     features_orm = validate_features(features)
@@ -71,29 +76,30 @@ def __init__(self, *args, **kwargs):
     else:
         type = float
     n_features = len(features)
-    features_hash = hash_set({feature.id for feature in features})
-    if id is None:
-        feature_set = FeatureSet.select(id=features_hash).one_or_none()
+    if hash is None:
+        features_hash = hash_set({feature.id for feature in features})
+        feature_set = FeatureSet.select(hash=features_hash).one_or_none()
         if feature_set is not None:
-            logger.info("Loaded an existing `feature_set`")
+            logger.info(f"Loaded {feature_set}")
             init_self_from_db(self, feature_set)
             return None
         else:
-            id = features_hash
+            hash = features_hash
     self._features = (get_related_name(features_orm), features)
     if type is not None:
         type_str = type.__name__ if not isinstance(type, str) else type
     else:
         type_str = None
     super(FeatureSet, self).__init__(
-        id=id,
+        id=ids.base62_20(),
         name=name,
         type=type_str,
         n=n_features,
-        readout=readout,
+        modality=modality,
         ref_orm=features_orm.__name__,
         ref_schema=features_orm.__get_schema_name__(),
         ref_field=ref_field,
+        hash=hash,
     )
 
 
@@ -121,7 +127,7 @@ def from_values(
     field: Field = Feature.name,
     type: Optional[Union[Type, str]] = None,
     name: Optional[str] = None,
-    readout: Optional[str] = None,
+    modality: Optional[str] = None,
     **kwargs,
 ) -> "FeatureSet":
     """{}"""
@@ -135,11 +141,10 @@ def from_values(
     iterable_idx = index_iterable(values)
     if not isinstance(iterable_idx[0], (str, int)):
         raise TypeError("values should be list-like of str or int")
-    n_features = len(iterable_idx)
     features_hash = hash_set(set(iterable_idx))
-    feature_set = FeatureSet.select(id=features_hash).one_or_none()
+    feature_set = FeatureSet.select(hash=features_hash).one_or_none()
     if feature_set is not None:
-        logger.info("Returning an existing feature_set")
+        logger.info(f"Loaded {feature_set}")
     else:
         from_bionty = ORM.__module__.startswith("lnschema_bionty")
         records = get_or_create_records(
@@ -150,13 +155,12 @@ def from_values(
         )
         # type_str = type.__name__ if not isinstance(type, str) else type
         feature_set = FeatureSet(
-            id=features_hash,
+            features=records,
+            hash=features_hash,
             name=name,
-            n=n_features,
-            readout=readout,
+            modality=modality,
             type=type,
             ref_field=field.field.name,
-            features=records,
         )
     return feature_set
 

lamindb/_file.py

@@ -17,6 +17,7 @@ from lnschema_core import Feature, FeatureSet, File, Run, ids
 from lnschema_core.types import AnnDataLike, DataLike, PathLike
 
 from lamindb._context import context
+from lamindb.dev import FeatureManager
 from lamindb.dev._settings import settings
 from lamindb.dev.hashing import b16_to_b64, hash_file
 from lamindb.dev.storage import (
@@ -353,6 +354,19 @@ def data_is_anndata(data: DataLike):
     return False
 
 
+def data_is_mudata(data: DataLike):
+    try:
+        from mudata import MuData
+    except ModuleNotFoundError:
+        return False
+
+    if isinstance(data, MuData):
+        return True
+    if isinstance(data, (str, Path, UPath)):
+        return Path(data).suffix in {".h5mu"}
+    return False
+
+
 def __init__(file: File, *args, **kwargs):
     # Below checks for the Django-internal call in from_db()
     # it'd be better if we could avoid this, but not being able to create a File
@@ -382,9 +396,7 @@ def __init__(file: File, *args, **kwargs):
     )
 
     if not len(kwargs) == 0:
-        raise ValueError(
-            "Only data, key, run, description & feature_sets can be passed."
-        )
+        raise ValueError("Only data, key, run, description can be passed.")
 
     if name is not None and description is not None:
         raise ValueError("Only pass description, do not pass a name")
@@ -392,19 +404,8 @@ def __init__(file: File, *args, **kwargs):
         logger.warning("Argument `name` is deprecated, please use `description`")
         description = name
 
-    if isinstance(data, pd.DataFrame) and log_hint:
-        logger.hint(
-            "This is a dataframe, consider using File.from_df() to link column"
-            " names as features!"
-        )
-    elif data_is_anndata(data) and log_hint:
-        logger.hint(
-            "This is AnnDataLike, consider using File.from_anndata() to link var_names"
-            " and obs.columns as features!"
-        )
-
     provisional_id = ids.base62_20()
-    kwargs, privates = get_file_kwargs_from_data(
+    kwargs_or_file, privates = get_file_kwargs_from_data(
         data=data,
         key=key,
         run=run,
@@ -412,17 +413,38 @@ def __init__(file: File, *args, **kwargs):
         provisional_id=provisional_id,
         skip_check_exists=skip_check_exists,
     )
+
     # an object with the same hash already exists
-    if isinstance(kwargs, File):
+    if isinstance(kwargs_or_file, File):
         # this is the way Django instantiates from the DB internally
         # https://github.com/django/django/blob/549d6ffeb6d626b023acc40c3bb2093b4b25b3d6/django/db/models/base.py#LL488C1-L491C51
         new_args = [
-            getattr(kwargs, field.attname) for field in file._meta.concrete_fields
+            getattr(kwargs_or_file, field.attname)
+            for field in file._meta.concrete_fields
         ]
         super(File, file).__init__(*new_args)
         file._state.adding = False
         file._state.db = "default"
         return None
+    else:
+        kwargs = kwargs_or_file
+
+    if isinstance(data, pd.DataFrame):
+        if log_hint:
+            logger.hint(
+                "This is a dataframe, consider using File.from_df() to link column"
+                " names as features!"
+            )
+        kwargs["accessor"] = "DataFrame"
+    elif data_is_anndata(data):
+        if log_hint:
+            logger.hint(
+                "This is AnnDataLike, consider using File.from_anndata() to link"
+                " var_names and obs.columns as features!"
+            )
+        kwargs["accessor"] = "AnnData"
+    elif data_is_mudata(data):
+        kwargs["accessor"] = "MuData"
 
     kwargs["id"] = provisional_id
     kwargs["description"] = description
@@ -468,7 +490,7 @@ def from_df(
     """{}"""
     file = File(data=df, key=key, run=run, description=description, log_hint=False)
     feature_set = FeatureSet.from_df(df)
-    file._feature_sets = [feature_set]
+    file._feature_sets = {"columns": feature_set}
     return file
 
 
@@ -497,19 +519,22 @@ def from_anndata(
         type = "float"
     else:
         type = convert_numpy_dtype_to_lamin_feature_type(adata.X.dtype)
-    feature_sets = []
-    logger.info("Parsing features of X (numerical)")
+    feature_sets = {}
+    logger.info("Parsing feature names of X, stored in slot .var")
     logger.indent = "   "
     feature_set_x = FeatureSet.from_values(
-        data_parse.var.index, var_ref, type=type, name="var", readout="abundance"
+        data_parse.var.index,
+        var_ref,
+        type=type,
     )
-    feature_sets.append(feature_set_x)
-    logger.indent = ""
-    logger.info("Parsing features of obs (numerical & categorical)")
-    logger.indent = "   "
-    feature_set_obs = FeatureSet.from_df(data_parse.obs, name="obs")
-    feature_sets.append(feature_set_obs)
+    feature_sets["var"] = feature_set_x
     logger.indent = ""
+    if len(data_parse.obs.columns) > 0:
+        logger.info("Parsing feature names of slot .obs")
+        logger.indent = "   "
+        feature_set_obs = FeatureSet.from_df(data_parse.obs)
+        feature_sets["obs"] = feature_set_obs
+        logger.indent = ""
     file._feature_sets = feature_sets
     return file
 
@@ -521,19 +546,13 @@ def from_dir(
     path: PathLike,
     *,
     run: Optional[Run] = None,
+    storage_root: Optional[PathLike] = None,
 ) -> List["File"]:
     """{}"""
     folderpath = UPath(path)
-    check_path_in_storage = check_path_in_default_storage(folderpath)
-
-    if check_path_in_storage:
-        folder_key = get_relative_path_to_root(path=folderpath).as_posix()
-    else:
-        raise RuntimeError(
-            "Currently, only directories in default storage can be registered!\n"
-            "You can either move your folder into the current default storage"
-            "or add a new default storage through `ln.settings.storage`"
-        )
+    folder_key = get_relative_path_to_root(
+        path=folderpath, root=storage_root
+    ).as_posix()
     # always sanitize by stripping a trailing slash
     folder_key = folder_key.rstrip("/")
     logger.hint(f"using storage prefix = {folder_key}/")
@@ -634,9 +653,14 @@ def _track_run_input(file: File, is_run_input: Optional[bool] = None):
             # avoid cycles (a file is both input and output)
             if file.run != context.run:
                 if settings.track_run_inputs:
+                    transform_note = ""
+                    if file.transform is not None:
+                        transform_note = (
+                            f", adding parent transform {file.transform.id}"
+                        )
                     logger.info(
-                        f"Adding file {file.id} as input for run {context.run.id},"
-                        f" adding parent transform {file.transform.id}"
+                        f"Adding file {file.id} as input for run"
+                        f" {context.run.id}{transform_note}"
                     )
                     track_run_input = True
                 else:
@@ -716,14 +740,21 @@ def _save_skip_storage(file, *args, **kwargs) -> None:
     if file.run is not None:
         file.run.save()
     if hasattr(file, "_feature_sets"):
-        for feature_set in file._feature_sets:
+        for feature_set in file._feature_sets.values():
             feature_set.save()
-    if hasattr(file, "_feature_values"):
-        for feature_value in file._feature_values:
-            feature_value.save()
     super(File, file).save(*args, **kwargs)
     if hasattr(file, "_feature_sets"):
-        file.feature_sets.set(file._feature_sets)
+        links = []
+        for slot, feature_set in file._feature_sets.items():
+            links.append(
+                File.feature_sets.through(
+                    file_id=file.id, feature_set_id=feature_set.id, slot=slot
+                )
+            )
+
+        from lamindb._save import bulk_create
+
+        bulk_create(links)
 
 
 def path(self) -> Union[Path, UPath]:
@@ -820,6 +851,9 @@ def inherit_relations(self, file: File, fields: Optional[List[str]] = None):
             else:
                 raise KeyError(f"No many-to-many relationship is found with '{field}'")
 
+    if None in related_names:
+        related_names.remove(None)
+
     inherit_names = [
         related_name
         for related_name in related_names
@@ -834,6 +868,15 @@ def inherit_relations(self, file: File, fields: Optional[List[str]] = None):
         )
 
 
+@property  # type: ignore
+@doc_args(File.features.__doc__)
+def features(self) -> "FeatureManager":
+    """{}"""
+    from lamindb._feature_manager import FeatureManager
+
+    return FeatureManager(self)
+
+
 METHOD_NAMES = [
     "__init__",
     "from_anndata",
@@ -864,5 +907,8 @@ for name in METHOD_NAMES:
 # privates currently dealt with separately
 File._delete_skip_storage = _delete_skip_storage
 File._save_skip_storage = _save_skip_storage
+# TODO: move these to METHOD_NAMES
 setattr(File, "view_lineage", view_lineage)
 setattr(File, "inherit_relations", inherit_relations)
+# property signature is not tested:
+setattr(File, "features", features)