lamindb 0.48a2__py3-none-any.whl → 0.48.1__py3-none-any.whl

lamindb/__init__.py

@@ -4,11 +4,12 @@ Import the package::
 
    import lamindb as ln
 
-`File` tracks data artifacts in form of files, on-disk (`zarr`, etc.) and
-in-memory data objects (`DataFrame`, `AnnData`, etc.) and allows to link them
-against entities of core schema & custom schemas.
+.. note::
 
-The core schema entities are central to lamindb's API:
+    `File` abstracts over objects in storage from blob-like files (pdf, txt, etc.)
+    to streamable storage backends (HDF5, DuckDB, zarr, TileDB, etc.).
+
+    `Dataset` abstracts over `File` and tables in classical warehouses (BigQuery, Snowflake).
 
 .. autosummary::
    :toctree: .
@@ -16,20 +17,13 @@ The core schema entities are central to lamindb's API:
    File
    Dataset
    Transform
-   Run
+   Label
    Feature
+   FeatureSet
+   Modality
    User
    Storage
-   Tag
-   Project
-
-More control over feature management:
-
-.. autosummary::
-   :toctree: .
-
-   FeatureSet
-   Category
+   Run
 
 Functional tools:
 
@@ -39,7 +33,6 @@ Functional tools:
    track
    view
    save
-   delete
 
 Static classes & modules:
 
@@ -47,7 +40,6 @@ Static classes & modules:
    :toctree: .
 
    settings
-   context
    types
    setup
    schema
@@ -55,14 +47,14 @@ Static classes & modules:
 
 """
 
-__version__ = "0.48a2"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.48.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 
 import lamindb_setup as _lamindb_setup
 
 # prints warning of python versions
-from lamin_logger import py_version_warning as _py_version_warning
+from lamin_utils import py_version_warning as _py_version_warning
 from lamindb_setup import _check_instance_setup
 from lamindb_setup._check_instance_setup import _INSTANCE_NOT_SETUP_WARNING
 
@@ -90,15 +82,14 @@ if _INSTANCE_SETUP:
     del InstanceNotSetupError
     del __getattr__  # delete so that imports work out
     from lnschema_core import (  # noqa
-        Category,
         Dataset,
         Feature,
         FeatureSet,
         File,
-        Project,
+        Label,
+        Modality,
         Run,
         Storage,
-        Tag,
         Transform,
         User,
     )
@@ -109,13 +100,13 @@ if _INSTANCE_SETUP:
     from ._context import context  # noqa
 
     track = context._track  # noqa
-    from lamin_logger import logger as _logger
+    from lamin_utils import logger as _logger
 
-    from . import _category  # noqa
     from . import _dataset  # noqa
     from . import _feature  # noqa
     from . import _feature_set  # noqa
     from . import _file  # noqa
+    from . import _label  # noqa
     from . import _orm  # noqa
     from . import _transform  # noqa
     from ._delete import delete  # noqa

lamindb/_context.py

@@ -6,7 +6,7 @@ from pathlib import Path, PurePath
 from typing import Dict, List, Optional, Tuple, Union
 
 import lnschema_core
-from lamin_logger import logger
+from lamin_utils import logger
 from lamindb_setup import settings
 from lamindb_setup.dev import InstanceSettings
 from lnschema_core import Run, Transform
@@ -116,7 +116,7 @@ def get_notebook_name_colab() -> str:
     return name.rstrip(".ipynb")
 
 
-class context:
+class run_context:
     """Global run context."""
 
     instance: Optional[InstanceSettings] = None
@@ -434,3 +434,6 @@ class context:
                     logger.success(f"Updated: {transform}")
 
         cls.transform = transform
+
+
+context = run_context

lamindb/_dataset.py

@@ -67,7 +67,7 @@ def from_files(dataset: Dataset, *, name: str, files: Iterable[File]) -> Dataset
     feature_set_file_links = File.feature_sets.through.objects.filter(
         file_id__in=file_ids
     )
-    feature_set_ids = [link.featureset_id for link in feature_set_file_links]
+    feature_set_ids = [link.feature_set_id for link in feature_set_file_links]
     feature_sets = FeatureSet.select(id__in=feature_set_ids)
     # validate consistency of feature_sets
     # we only allow one feature set per type
@@ -128,13 +128,16 @@ def delete(dataset: Dataset, storage: bool = False):
 def save(dataset: Dataset):
     if dataset.file is not None:
         dataset.file.save()
-    for feature_set in dataset._feature_sets:
+    feature_sets = dataset._feature_sets
+    if isinstance(feature_sets, dict):
+        feature_sets = feature_sets.values()
+    for feature_set in feature_sets:
         feature_set.save()
     super(Dataset, dataset).save()
     if len(dataset._files) > 0:
         dataset.files.set(dataset._files)
     if len(dataset._feature_sets) > 0:
-        dataset.feature_sets.set(dataset._feature_sets)
+        dataset.feature_sets.set(feature_sets)
 
 
 Dataset.__init__ = __init__

lamindb/_delete.py

@@ -1,6 +1,6 @@
 from typing import List, Union, overload  # noqa
 
-from lamin_logger import colors, logger
+from lamin_utils import colors, logger
 from lnschema_core import ORM
 
 
@@ -47,12 +47,12 @@ def delete(  # type: ignore
 
         Bulk delete via QuerySet:
 
-        >>> ln.save(ln.Tag.from_values(["Tag1", "Tag2", "Tag3"], field="name"))
-        >>> queryset = ln.Tag.select(name__icontains = "tag")
+        >>> ln.save(ln.Label.from_values(["Label1", "Label2", "Label3"], field="name"))
+        >>> queryset = ln.Label.select(name__icontains = "label")
         >>> queryset.list()
-        [Tag(id=o3FY3c5n, name=Tag2, updated_at=2023-07-19 18:28:16, created_by_id=kmvZDIX9), # noqa
-        Tag(id=Qi3c4utq, name=Tag3, updated_at=2023-07-19 18:28:16, created_by_id=kmvZDIX9), # noqa
-        Tag(id=CcFPLmpq, name=Tag1, updated_at=2023-07-19 18:28:16, created_by_id=kmvZDIX9)] # noqa
+        [Label(id=o3FY3c5n, name=Label2, updated_at=2023-07-19 18:28:16, created_by_id=kmvZDIX9), # noqa
+        Label(id=Qi3c4utq, name=Label3, updated_at=2023-07-19 18:28:16, created_by_id=kmvZDIX9), # noqa
+        Label(id=CcFPLmpq, name=Label1, updated_at=2023-07-19 18:28:16, created_by_id=kmvZDIX9)] # noqa
         >>> queryset.delete()
     """
     logger.warning("For efficient bulk delete, use `queryset.delete` instead")

lamindb/_feature.py

@@ -1,8 +1,10 @@
-from typing import List
+from itertools import islice
+from typing import List, Optional, Union
 
 import pandas as pd
+from lamin_utils import logger
 from lamindb_setup.dev._docs import doc_args
-from lnschema_core import Category, Feature
+from lnschema_core import Feature, Label
 from pandas.api.types import is_categorical_dtype, is_string_dtype
 
 from lamindb.dev.utils import attach_func_to_class_method
@@ -11,6 +13,18 @@ from . import _TESTING
 from ._save import bulk_create
 
 
+def convert_numpy_dtype_to_lamin_feature_type(dtype) -> str:
+    orig_type = dtype.name
+    # strip precision qualifiers
+    type = "".join(i for i in orig_type if not i.isdigit())
+    return type
+
+
+def take(n, iterable):
+    """Return the first n items of the iterable as a list."""
+    return list(islice(iterable, n))
+
+
 def __init__(self, *args, **kwargs):
     if len(args) == len(self._meta.concrete_fields):
         super(Feature, self).__init__(*args, **kwargs)
@@ -18,43 +32,64 @@ def __init__(self, *args, **kwargs):
     # now we proceed with the user-facing constructor
     if len(args) != 0:
         raise ValueError("Only non-keyword args allowed")
+    type: Optional[Union[type, str]] = kwargs.pop("type") if "type" in kwargs else None
+    if type is not None:
+        type_str = type.__name__ if not isinstance(type, str) else type
+    else:
+        type_str = None
+    kwargs["type"] = type_str
     super(Feature, self).__init__(*args, **kwargs)
 
 
 @classmethod  # type:ignore
 @doc_args(Feature.from_df.__doc__)
-def from_df(cls, df) -> List["Feature"]:
+def from_df(cls, df: "pd.DataFrame") -> List["Feature"]:
     """{}"""
-    records = Feature.from_values(df.columns, field=Feature.name)
-    assert len(records) == len(df.columns)
-
     string_cols = [col for col in df.columns if is_string_dtype(df[col])]
     categoricals = {col: df[col] for col in df.columns if is_categorical_dtype(df[col])}
     for key in string_cols:
         c = pd.Categorical(df[key])
-        # TODO: We should only check if non-null values are unique, but
-        # this would break cases where string columns with nulls could
-        # be written as categorical, but not as string.
-        # Possible solution: https://github.com/scverse/anndata/issues/504
         if len(c.categories) < len(c):
             categoricals[key] = c
 
-    for record in records:
-        if record.name in categoricals:
-            record.type = "Category"
-            feature = Feature.select(name=record.name).one_or_none()
-            categories = categoricals[record.name].categories
-            if feature is not None:
-                record._categories_records = Category.from_values(
-                    categories, feature=feature
-                )
-            else:
-                record._categories_raw = categories
+    types = {}
+    categoricals_with_unmapped_categories = {}
+    for name, col in df.items():
+        if name in categoricals:
+            types[name] = "category"
+            categorical = categoricals[name]
+            if hasattr(
+                categorical, "cat"
+            ):  # because .categories > pd2.0, .cat.categories < pd2.0
+                categorical = categorical.cat
+            categories = categorical.categories
+            categoricals_with_unmapped_categories[name] = Label.select(
+                feature=name
+            ).inspect(categories, "name", logging=False)["not_mapped"]
         else:
-            orig_type = df[record.name].dtype.name
-            # strip precision qualifiers
-            record.type = "".join(i for i in orig_type if not i.isdigit())
-    return records
+            types[name] = convert_numpy_dtype_to_lamin_feature_type(col.dtype)
+
+    features = Feature.from_values(df.columns, field=Feature.name, types=types)
+    assert len(features) == len(df.columns)
+
+    if len(categoricals_with_unmapped_categories) > 0:
+        n_max = 20
+        categoricals_with_unmapped_categories_formatted = "\n      ".join(
+            [
+                f"{key}: {', '.join(value)}"
+                for key, value in take(
+                    n_max, categoricals_with_unmapped_categories.items()
+                )
+            ]
+        )
+        if len(categoricals_with_unmapped_categories) > n_max:
+            categoricals_with_unmapped_categories_formatted += "\n      ..."
+        categoricals_with_unmapped_categories_formatted
+        logger.info(
+            "There are unmapped categories:\n     "
+            f" {categoricals_with_unmapped_categories_formatted}"
+        )
+    return features
 
 
 @doc_args(Feature.save.__doc__)
@@ -65,7 +100,7 @@ def save(self, *args, **kwargs) -> None:
     if hasattr(self, "_categories_records"):
         records = self._categories_records
     if hasattr(self, "_categories_raw"):
-        records = Category.from_values(self._categories_raw, feature=self)
+        records = Label.from_values(self._categories_raw, feature=self)
     if records is not None:
         bulk_create(records)
 

lamindb/_feature_manager.py

@@ -0,0 +1,176 @@
+from collections import defaultdict
+from typing import Iterable, List, Optional, Union
+
+import pandas as pd
+from lamin_utils import logger
+from lnschema_core.models import ORM, Dataset, Feature, FeatureSet, File
+
+from ._queryset import QuerySet
+from ._save import save
+
+
+def validate_and_cast_feature(feature) -> Feature:
+    if isinstance(feature, str):
+        feature_name = feature
+        feature = Feature.select(name=feature_name).one_or_none()
+        if feature is None:
+            raise ValueError(
+                f"Please create feature: ln.Feature(name='{feature_name}',"
+                " type='category').save()"
+            )
+    return feature
+
+
+def create_features_df(
+    file: File, feature_sets: List[FeatureSet], exclude: bool = True
+):
+    features = []
+    for feature_set in feature_sets:
+        if exclude:
+            features_df = feature_set.features.exclude(labels_orm__isnull=True).df()
+        else:
+            features_df = feature_set.features.df()
+        slots = file.feature_sets.through.objects.filter(
+            file=file, feature_set=feature_set
+        ).list("slot")
+        for slot in slots:
+            features_df["slot"] = slot
+            features.append(features_df)
+    features_df = pd.concat(features)
+    return features_df.sort_values(["labels_schema", "labels_orm"])
+
+
+class FeatureManager:
+    """Feature manager."""
+
+    def __init__(self, host: Union[File, Dataset]):
+        self._host = host
+        slot_feature_sets = (
+            self._feature_set_df_with_slots().reset_index().set_index("slot")["id"]
+        )
+        self._slots = {
+            slot: self._host.feature_sets.get(id=i)
+            for slot, i in slot_feature_sets.items()
+        }
+
+    def __repr__(self) -> str:
+        if len(self._slots) > 0:
+            msg = "slots:\n"
+            for slot, feature_set in self._slots.items():
+                msg += f"    {slot}: {feature_set}\n"
+            return msg
+        else:
+            return "No linked features."
+
+    def __getitem__(self, slot) -> QuerySet:
+        id = (
+            self._host.feature_sets.through.objects.filter(
+                file_id=self._host.id, slot=slot
+            )
+            .one()
+            .feature_set_id
+        )
+        accessor_by_orm = {
+            field.related_model.__name__: field.name
+            for field in self._host._meta.related_objects
+        }
+        accessor_by_orm["Feature"] = "features"
+        feature_set = self._host.feature_sets.filter(id=id).one()
+        return getattr(feature_set, accessor_by_orm[feature_set.ref_orm]).all()
+
+    def _feature_set_df_with_slots(self) -> pd.DataFrame:
+        """Return DataFrame."""
+        df = self._host.feature_sets.df()
+        df.insert(
+            0,
+            "slot",
+            self._host.feature_sets.through.objects.filter(file_id=self._host.id)
+            .df()
+            .set_index("feature_set_id")
+            .slot,
+        )
+        return df
+
+    def add_labels(
+        self, records: Union[ORM, List[ORM]], feature: Optional[Union[str, ORM]] = None
+    ):
+        """Add one or several labels and associate them with a feature."""
+        if isinstance(records, str) or not isinstance(records, Iterable):
+            records = [records]
+        if isinstance(records[0], str):  # type: ignore
+            raise ValueError(
+                "Please pass a record (an ORM object), not a string, e.g., via: label"
+                f" = ln.Label(name='{records[0]}')"  # type: ignore
+            )
+        if self._host._state.adding:
+            raise ValueError("Please save the file or dataset before adding a label!")
+        feature = validate_and_cast_feature(feature)
+        records_by_orm = defaultdict(list)
+        records_by_feature_orm = defaultdict(list)
+        for record in records:
+            records_by_orm[record.__class__.__name__].append(record)
+            if feature is None:
+                try:
+                    record_feature = (
+                        record._feature
+                        if hasattr(record, "_feature")
+                        else record.feature
+                    )
+                except ValueError:
+                    raise ValueError("Pass feature argument")
+            else:
+                record_feature = feature
+            records_by_feature_orm[(record_feature, record.__class__.__name__)].append(
+                record
+            )
+        schema_and_accessor_by_orm = {
+            field.related_model.__name__: (
+                field.related_model.__get_schema_name__(),
+                field.name,
+            )
+            for field in self._host._meta.related_objects
+        }
+        schema_and_accessor_by_orm["Label"] = ("core", "labels")
+        for orm_name, records in records_by_orm.items():
+            save(records)
+            getattr(self._host, schema_and_accessor_by_orm[orm_name][1]).set(records)
+        accessor_by_orm = {
+            field.related_model.__name__: field.name
+            for field in self._host._meta.related_objects
+        }
+        accessor_by_orm["Feature"] = "features"
+        feature_sets = self._host.feature_sets.all()
+        feature_sets_by_orm = {
+            feature_set.ref_orm: feature_set for feature_set in feature_sets
+        }
+        for (feature, orm_name), records in records_by_feature_orm.items():
+            feature = validate_and_cast_feature(feature)
+            logger.info(f"Linking feature {feature.name} to {orm_name}")
+            feature.labels_orm = orm_name
+            feature.labels_schema = schema_and_accessor_by_orm[orm_name][0]
+            feature.save()
+            # check whether we have to update the feature set that manages labels
+            # (Feature) to account for a new feature
+            feature_set = feature_sets_by_orm["Feature"]
+            accessor = "features"
+            linked_features = getattr(feature_set, accessor)
+            if feature not in linked_features.all():
+                logger.info(
+                    f"Linking feature {feature.name} to feature set {feature_set}"
+                )
+                linked_features.add(feature)
+                feature_set.n += 1
+                feature_set.save()
+
+    def add_feature_set(self, feature_set: FeatureSet, slot: str):
+        if self._host._state.adding:
+            raise ValueError(
+                "Please save the file or dataset before adding a feature set!"
+            )
+        feature_set.save()
+        self._host.feature_sets.add(feature_set)
+        link_record = self._host.feature_sets.through.objects.filter(
+            file=self._host, feature_set=feature_set
+        ).one()
+        link_record.slot = slot
+        link_record.save()

lamindb/_feature_set.py

@@ -1,10 +1,10 @@
-from typing import List, Optional
+from typing import Iterable, List, Optional, Type, Union
 
 import pandas as pd
 from django.db.models.query_utils import DeferredAttribute as Field
-from lamin_logger import logger
+from lamin_utils import logger
 from lamindb_setup.dev._docs import doc_args
-from lnschema_core import ORM, Feature, FeatureSet
+from lnschema_core import ORM, Feature, FeatureSet, ids
 from lnschema_core.types import ListLike
 
 from lamindb.dev.hashing import hash_set
@@ -55,28 +55,51 @@ def __init__(self, *args, **kwargs):
     # now we proceed with the user-facing constructor
     if len(args) > 1:
         raise ValueError("Only one non-keyword arg allowed: features")
-    features: List[ORM] = kwargs.pop("features") if len(args) == 0 else args[0]
-    field: Optional[str] = kwargs.pop("field") if "field" in kwargs else None
-    id: Optional[str] = kwargs.pop("id") if "id" in kwargs else None
-    features_type = validate_features(features)
-    related_name = get_related_name(features_type)
-    if id is None:
+    features: Iterable[ORM] = kwargs.pop("features") if len(args) == 0 else args[0]
+    ref_field: Optional[str] = (
+        kwargs.pop("ref_field") if "ref_field" in kwargs else "id"
+    )
+    type: Optional[Union[type, str]] = kwargs.pop("type") if "type" in kwargs else None
+    modality: Optional[str] = kwargs.pop("modality") if "modality" in kwargs else None
+    name: Optional[str] = kwargs.pop("name") if "name" in kwargs else None
+    # hash is only internally used
+    hash: Optional[str] = kwargs.pop("hash") if "hash" in kwargs else None
+    if len(kwargs) > 0:
+        raise ValueError(
+            "Only features, ref_field, type, modality, name are valid keyword arguments"
+        )
+
+    # now code
+    features_orm = validate_features(features)
+    if features_orm == Feature:
+        type = None
+    else:
+        type = float
+    n_features = len(features)
+    if hash is None:
         features_hash = hash_set({feature.id for feature in features})
-        feature_set = FeatureSet.select(id=features_hash).one_or_none()
+        feature_set = FeatureSet.select(hash=features_hash).one_or_none()
         if feature_set is not None:
-            logger.info("Returning an existing feature_set")
+            logger.info(f"Loaded {feature_set}")
             init_self_from_db(self, feature_set)
             return None
         else:
-            id = features_hash
-    self._features = (related_name, features)
-    if field is None:
-        field = "id"
+            hash = features_hash
+    self._features = (get_related_name(features_orm), features)
+    if type is not None:
+        type_str = type.__name__ if not isinstance(type, str) else type
+    else:
+        type_str = None
     super(FeatureSet, self).__init__(
-        id=id,
-        type=features_type.__name__,
-        schema=features_type.__get_schema_name__(),
-        field=field,
+        id=ids.base62_20(),
+        name=name,
+        type=type_str,
+        n=n_features,
+        modality=modality,
+        ref_orm=features_orm.__name__,
+        ref_schema=features_orm.__get_schema_name__(),
+        ref_field=ref_field,
+        hash=hash,
     )
 
 
@@ -99,33 +122,45 @@ def save(self, *args, **kwargs) -> None:
 @classmethod  # type:ignore
 @doc_args(FeatureSet.from_values.__doc__)
 def from_values(
-    cls, values: ListLike, field: Field = Feature.name, **kwargs
+    cls,
+    values: ListLike,
+    field: Field = Feature.name,
+    type: Optional[Union[Type, str]] = None,
+    name: Optional[str] = None,
+    modality: Optional[str] = None,
+    **kwargs,
 ) -> "FeatureSet":
     """{}"""
     if not isinstance(field, Field):
         raise TypeError("Argument `field` must be an ORM field, e.g., `Feature.name`")
     if len(values) == 0:
         raise ValueError("Provide a list of at least one value")
-    orm = field.field.model
+    ORM = field.field.model
+    if isinstance(ORM, Feature):
+        raise ValueError("Please use from_df() instead of from_values()")
     iterable_idx = index_iterable(values)
     if not isinstance(iterable_idx[0], (str, int)):
         raise TypeError("values should be list-like of str or int")
     features_hash = hash_set(set(iterable_idx))
-    feature_set = FeatureSet.select(id=features_hash).one_or_none()
+    feature_set = FeatureSet.select(hash=features_hash).one_or_none()
     if feature_set is not None:
-        logger.info("Returning an existing feature_set")
+        logger.info(f"Loaded {feature_set}")
     else:
-        from_bionty = orm.__module__.startswith("lnschema_bionty")
+        from_bionty = ORM.__module__.startswith("lnschema_bionty")
         records = get_or_create_records(
             iterable=iterable_idx,
             field=field,
             from_bionty=from_bionty,
             **kwargs,
         )
+        # type_str = type.__name__ if not isinstance(type, str) else type
         feature_set = FeatureSet(
-            id=features_hash,
-            field=field.field.name,
             features=records,
+            hash=features_hash,
+            name=name,
+            modality=modality,
+            type=type,
+            ref_field=field.field.name,
         )
     return feature_set
 
@@ -135,10 +170,11 @@ def from_values(
 def from_df(
     cls,
     df: "pd.DataFrame",
+    name: Optional[str] = None,
 ) -> "FeatureSet":
     """{}"""
     features = Feature.from_df(df)
-    feature_set = FeatureSet(features)
+    feature_set = FeatureSet(features, name=name)
     return feature_set