lamindb 0.49.3__py3-none-any.whl → 0.50.1__py3-none-any.whl

lamindb/__init__.py

@@ -1,29 +1,67 @@
-"""Open-source data lake & feature store for biology.
+"""Open-source data platform for biology.
 
-Import the package::
+LaminDB helps you manage data using registries.
+The two most central are:
 
-   import lamindb as ln
+.. autosummary::
+   :toctree: .
+
+   File
+   Dataset
+
+
+.. dropdown::  With more detail, what are files & datasets?
+
+    Both files & datasets
+
+    - track numerical & categorical data batches of arbitrary format & size
+
+    - can validate & link features (the measured dimensions in a data batch)
+
+    Roughly,
+
+    - a file stores a single immutable batch of data
+
+    - a dataset stores a mutable collection of data batches
+
+    Examples:
+
+    - Blob-like immutable files (pdf, txt, csv, jpg, ...) or arrays (h5, h5ad,
+      ...) → :class:`~lamindb.File`
+
+    - Mutable streamable backends (DuckDB, zarr, TileDB, ...) → :class:`~lamindb.Dataset` wrapping :class:`~lamindb.File`
+
+    - Collections of files → :class:`~lamindb.Dataset` wrapping :class:`~lamindb.File`
 
-.. note::
+    - Datasets in BigQuery, Snowflake, Postgres, ... → :class:`~lamindb.Dataset` (not yet implemented)
 
-    `File` abstracts over objects in storage from blob-like files (pdf, txt, etc.)
-    to streamable storage backends (HDF5, DuckDB, zarr, TileDB, etc.).
+    Hence, while
 
-    `Dataset` abstracts over `File` and tables in classical warehouses (BigQuery, Snowflake).
+    - files *always* have a one-to-one correspondence with a storage accessor
+
+    - datasets *can* reference a single file, multiple files or a dataset
+      in a warehouse like BigQuery or Snowflake
+
+
+There are four registries to track provenance of data batches:
 
 .. autosummary::
    :toctree: .
 
-   File
-   Dataset
+   User
+   Storage
    Transform
+   Run
+
+And four registries to validate & contextualize measurements in data batches:
+
+.. autosummary::
+   :toctree: .
+
    Label
    Feature
    FeatureSet
    Modality
-   User
-   Storage
-   Run
 
 Functional tools:
 
@@ -47,7 +85,7 @@ Static classes & modules:
 
 """
 
-__version__ = "0.49.3"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.50.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 
@@ -107,11 +145,13 @@ if _INSTANCE_SETUP:
     from . import _feature_set  # noqa
     from . import _file  # noqa
     from . import _label  # noqa
-    from . import _orm  # noqa
+    from . import _registry  # noqa
     from . import _storage  # noqa
+    from . import _synonym  # noqa
     from . import _transform  # noqa
+    from . import _validate  # noqa
     from ._delete import delete  # noqa
-    from ._orm import select_backward as select  # noqa
+    from ._registry import select_backward as select  # noqa
     from ._save import save  # noqa
     from ._view import view  # noqa
     from .dev._settings import settings

lamindb/_context.py

@@ -15,16 +15,16 @@ from lnschema_core.types import TransformType
 is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
 
 msg_path_failed = (
-    "Failed to infer notebook path.\nFix: Either track manually via"
+    "failed to infer notebook path.\nfix: either track manually via"
     " `ln.track(ln.Transform(name='My notebook'))` or pass"
-    " `notebook_path` to ln.track()."
+    " `notebook_path` to ln.track()"
 )
 
 msg_manual_init = (
-    "\n(1) Save your notebook!"
-    "\n(2) Attach metadata to the notebook by running the CLI:\n"
+    "\n(1) save your notebook!"
+    "\n(2) attach metadata to the notebook by running the CLI:\n"
     "lamin track {notebook_path}"
-    "\n(3) Reload or re-open your notebook"
+    "\n(3) reload or re-open your notebook"
 )
 
 
@@ -167,8 +167,8 @@ class run_context:
             install[jupyter]`, you can simply call:
 
             >>> ln.track()
-            ✅ Saved: Transform(id=1LCd8kco9lZUBg, name=Track data lineage / provenance, short_name=02-data-lineage, stem_id=1LCd8kco9lZU, version=0, type=notebook, updated_at=2023-07-10 18:37:19, created_by_id=DzTjkKse) # noqa
-            ✅ Saved: Run(id=pHgVICV9DxBaV6BAuKJl, run_at=2023-07-10 18:37:19, transform_id=1LCd8kco9lZUBg, created_by_id=DzTjkKse) # noqa
+            ✅ saved: Transform(id=1LCd8kco9lZUBg, name=Track data lineage / provenance, short_name=02-data-lineage, stem_id=1LCd8kco9lZU, version=0, type=notebook, updated_at=2023-07-10 18:37:19, created_by_id=DzTjkKse) # noqa
+            ✅ saved: Run(id=pHgVICV9DxBaV6BAuKJl, run_at=2023-07-10 18:37:19, transform_id=1LCd8kco9lZUBg, created_by_id=DzTjkKse) # noqa
             >>> ln.context.transform
             Transform(id=1LCd8kco9lZUBg, name=Track data lineage / provenance, short_name=02-data-lineage, stem_id=1LCd8kco9lZU, version=0, type=notebook, updated_at=2023-07-10 18:37:19, created_by_id=DzTjkKse) # noqa
             >>> ln.context.run
@@ -180,8 +180,8 @@ class run_context:
             >>> ln.Transform(name="Cell Ranger", version="7.2.0", type="pipeline").save()
             >>> transform = ln.Transform.filter(name="Cell Ranger", version="7.2.0").one()
             >>> ln.track(transform)
-            💬 Loaded: Transform(id=ceHkZMaiHFdoB6, name=Cell Ranger, stem_id=ceHkZMaiHFdo, version=7.2.0, type=pipeline, updated_at=2023-07-10 18:37:19, created_by_id=DzTjkKse) # noqa
-            ✅ Saved: Run(id=RcpWIKC8cF74Pn3RUJ1W, run_at=2023-07-10 18:37:19, transform_id=ceHkZMaiHFdoB6, created_by_id=DzTjkKse) # noqa
+            💬 loaded: Transform(id=ceHkZMaiHFdoB6, name=Cell Ranger, stem_id=ceHkZMaiHFdo, version=7.2.0, type=pipeline, updated_at=2023-07-10 18:37:19, created_by_id=DzTjkKse) # noqa
+            ✅ saved: Run(id=RcpWIKC8cF74Pn3RUJ1W, run_at=2023-07-10 18:37:19, transform_id=ceHkZMaiHFdoB6, created_by_id=DzTjkKse) # noqa
             >>> ln.context.transform
             Transform(id=ceHkZMaiHFdoB6, name=Cell Ranger, stem_id=ceHkZMaiHFdo, version=7.2.0, type=pipeline, updated_at=2023-07-10 18:37:19, created_by_id=DzTjkKse) # noqa
             >>> ln.context.run
@@ -204,20 +204,20 @@ class run_context:
                 except Exception as e:
                     if isinstance(e, ImportError):
                         logger.info(
-                            "It looks like you are running ln.track() from a "
-                            "notebook!\nPlease install nbproject: pip install nbproject"
+                            "it looks like you are running ln.track() from a "
+                            "notebook!\nplease install nbproject: pip install nbproject"
                         )
                     elif isinstance(e, UpdateNbWithNonInteractiveEditorError):
                         raise e
                     elif isinstance(e, (NotebookNotSavedError, NoTitleError)):
                         raise e
                     else:
-                        logger.warning(f"Automatic tracking of notebook failed: {e}")
+                        logger.warning(f"automatic tracking of notebook failed: {e}")
                     is_tracked_notebook = False
 
             if not is_tracked_notebook:
                 logger.warning(
-                    "No automatic metadata detection, consider passing transform"
+                    "no automatic metadata detection, consider passing transform"
                 )
                 return None
         else:
@@ -227,10 +227,10 @@ class run_context:
                 transform_exists = Transform.filter(id=transform.id).first()
             if transform_exists is None:
                 transform.save()
-                logger.success(f"Saved: {transform}")
+                logger.save(f"saved: {transform}")
                 transform_exists = transform
             else:
-                logger.info(f"Loaded: {transform_exists}")
+                logger.success(f"loaded: {transform_exists}")
             cls.transform = transform_exists
 
         if new_run is None:  # for notebooks, default to loading latest runs
@@ -248,22 +248,22 @@ class run_context:
             if run is not None:  # loaded latest run
                 run.run_at = datetime.now(timezone.utc)  # update run time
                 run.save()
-                logger.info(f"Loaded: {run}")
+                logger.success(f"loaded: {run}")
 
         if run is None:  # create new run
             run = ln.Run(transform=cls.transform)
             run.save()
-            logger.success(f"Saved: {run}")
+            logger.save(f"saved: {run}")
         cls.run = run
 
         # at this point, we have a transform can display its parents if there are any
         parents = cls.transform.parents.all() if cls.transform is not None else []
         if len(parents) > 0:
             if len(parents) == 1:
-                logger.info(f"Parent transform: {parents[0]}")
+                logger.info(f"  parent transform: {parents[0]}")
             else:
                 parents_formatted = "\n   - ".join([f"{parent}" for parent in parents])
-                logger.info(f"Parent transforms:\n   - {parents_formatted}")
+                logger.info(f"  parent transforms:\n   - {parents_formatted}")
 
         # only for newly intialized notebooks
         if hasattr(cls, "_notebook_meta"):
@@ -347,11 +347,11 @@ class run_context:
 
                 dm = DisplayMeta(metadata)
                 logger.info(
-                    "Notebook imports:"
+                    "notebook imports:"
                     f" {' '.join(dm.pypackage(infer_pypackages(nb, pin_versions=True)))}"  # noqa
                 )
             except Exception:
-                logger.debug("Inferring imported packages failed")
+                logger.debug("inferring imported packages failed")
                 pass
 
         if needs_init:
@@ -405,9 +405,9 @@ class run_context:
                 type=TransformType.notebook,
             )
             transform.save()
-            logger.success(f"Saved: {transform}")
+            logger.save(f"saved: {transform}")
         else:
-            logger.info(f"Loaded: {transform}")
+            logger.success(f"loaded: {transform}")
             if transform.name != title or transform.short_name != filestem:
                 response = input(
                     "Updated notebook name and/or title: Do you want to assign a"
@@ -427,9 +427,9 @@ class run_context:
                 transform.short_name = filestem
                 transform.save()
                 if response == "y":
-                    logger.success(f"Saved: {transform}")
+                    logger.save(f"saved: {transform}")
                 else:
-                    logger.success(f"Updated: {transform}")
+                    logger.success(f"updated: {transform}")
 
         cls.transform = transform
 

lamindb/_delete.py

@@ -1,30 +1,30 @@
 from typing import List, Union, overload  # noqa
 
 from lamin_utils import colors, logger
-from lnschema_core import ORM
+from lnschema_core import Registry
 
 
 @overload
 def delete(
-    record: ORM,
+    record: Registry,
 ) -> None:
     ...
 
 
 @overload
 def delete(
-    records: List[ORM],
+    records: List[Registry],
 ) -> None:  # type: ignore
     ...
 
 
 def delete(  # type: ignore
-    records: Union[ORM, List[ORM]],
+    records: Union[Registry, List[Registry]],
 ) -> None:
     """Delete metadata records & files.
 
     Args:
-        records: `Union[ORM, List[ORM]]` One or multiple records.
+        records: `Union[Registry, List[Registry]]` One or multiple records.
 
     Returns:
         `None`
@@ -55,11 +55,11 @@ def delete(  # type: ignore
         Label(id=CcFPLmpq, name=Label1, updated_at=2023-07-19 18:28:16, created_by_id=kmvZDIX9)] # noqa
         >>> queryset.delete()
     """
-    logger.warning("For efficient bulk delete, use `queryset.delete` instead")
+    logger.warning("for efficient bulk delete, use `queryset.delete` instead")
     if isinstance(records, list):
         records = records
-    elif isinstance(records, ORM):
+    elif isinstance(records, Registry):
         records = [records]
     for record in records:
         record.delete()
-        logger.success(f"Deleted {colors.yellow(f'{record}')}")
+        logger.success(f"deleted {colors.yellow(f'{record}')}")

lamindb/_feature.py

@@ -5,7 +5,7 @@ import pandas as pd
 from lamin_utils import colors, logger
 from lamindb_setup.dev._docs import doc_args
 from lnschema_core import Feature, Label
-from lnschema_core.models import ORM
+from lnschema_core.models import Registry
 from pandas.api.types import is_categorical_dtype, is_string_dtype
 
 from lamindb.dev.utils import attach_func_to_class_method
@@ -34,7 +34,7 @@ def __init__(self, *args, **kwargs):
     if len(args) != 0:
         raise ValueError("Only non-keyword args allowed")
     type: Optional[Union[type, str]] = kwargs.pop("type") if "type" in kwargs else None
-    registries: Optional[List[ORM]] = (
+    registries: Optional[List[Registry]] = (
         kwargs.pop("registries") if "registries" in kwargs else None
     )
     # cast type
@@ -46,14 +46,18 @@ def __init__(self, *args, **kwargs):
     # cast registries
     registries_str: Optional[str] = None
     if registries is not None:
-        if not isinstance(registries, List):
-            raise ValueError("registries has to be a list of ORM types")
-        registries_str = ""
-        for cls in registries:
-            if not hasattr(cls, "__get_name_with_schema__"):
-                raise ValueError("each element of the list has to be an ORM type")
-            registries_str += cls.__get_name_with_schema__() + "|"
-        registries_str = registries_str.rstrip("|")
+        if isinstance(registries, str):
+            # TODO: add more validation
+            registries_str = registries
+        else:
+            if not isinstance(registries, List):
+                raise ValueError("registries has to be a list of Registry types")
+            registries_str = ""
+            for cls in registries:
+                if not hasattr(cls, "__get_name_with_schema__"):
+                    raise ValueError("each element of the list has to be a Registry")
+                registries_str += cls.__get_name_with_schema__() + "|"
+            registries_str = registries_str.rstrip("|")
     kwargs["registries"] = registries_str
     super(Feature, self).__init__(*args, **kwargs)
 
@@ -75,7 +79,7 @@ def from_df(cls, df: "pd.DataFrame") -> List["Feature"]:
         if name in categoricals:
             types[name] = "category"
             # below is a harder feature to write, now, because it requires to
-            # query the link tables between the label ORM and file or dataset
+            # query the link tables between the label Registry and file or dataset
             # the original implementation fell short
             # categorical = categoricals[name]
             # if hasattr(

lamindb/_feature_set.py

@@ -4,7 +4,7 @@ import pandas as pd
 from django.db.models.query_utils import DeferredAttribute as Field
 from lamin_utils import logger
 from lamindb_setup.dev._docs import doc_args
-from lnschema_core import ORM, Feature, FeatureSet, Modality, ids
+from lnschema_core import Feature, FeatureSet, Modality, Registry, ids
 from lnschema_core.types import ListLike
 
 from lamindb.dev.hashing import hash_set
@@ -12,11 +12,11 @@ from lamindb.dev.utils import attach_func_to_class_method
 
 from . import _TESTING
 from ._from_values import get_or_create_records, index_iterable
-from ._orm import init_self_from_db
+from ._registry import init_self_from_db
 from ._save import bulk_create
 
 
-def get_related_name(features_type: ORM):
+def get_related_name(features_type: Registry):
     candidates = [
         field.related_name
         for field in FeatureSet._meta.related_objects
@@ -26,19 +26,20 @@ def get_related_name(features_type: ORM):
         raise ValueError(
             f"Can't create feature sets from {features_type.__name__} because it's not"
             " related to it!\nYou need to create a link model between FeatureSet and"
-            " your ORM in your custom schema.\nTo do so, add a line:\nfeature_sets ="
-            " models.ManyToMany(FeatureSet, related_name='mythings')\n"
+            " your Registry in your custom schema.\nTo do so, add a"
+            " line:\nfeature_sets = models.ManyToMany(FeatureSet,"
+            " related_name='mythings')\n"
         )
     return candidates[0]
 
 
-def validate_features(features: List[ORM]) -> ORM:
+def sanity_check_features(features: List[Registry]) -> Registry:
     """Validate and return feature type."""
     if len(features) == 0:
         raise ValueError("provide list of features with at least one element")
     if not hasattr(features, "__getitem__"):
         raise TypeError("features has to be list-like")
-    if not isinstance(features[0], ORM):
+    if not isinstance(features[0], Registry):
         raise TypeError(
             "features has to store feature records! use .from_values() otherwise"
         )
@@ -48,6 +49,24 @@ def validate_features(features: List[ORM]) -> ORM:
     return next(iter(feature_types))  # return value in set of cardinality 1
 
 
+def get_validated_features(features: List[Registry], field: Field) -> List[Registry]:
+    validated_features = []
+    non_validated_features = []
+    for feature in features:
+        if feature._state.adding and not (
+            hasattr(feature, "_from_bionty") and feature._from_bionty
+        ):
+            non_validated_features.append(getattr(feature, field.field.name))
+        else:
+            validated_features.append(feature)
+    if non_validated_features:
+        non_validated_features_display = ",".join(non_validated_features)
+        logger.warning(
+            f"ignoring non-validated features: {non_validated_features_display}"
+        )
+    return validated_features
+
+
 def __init__(self, *args, **kwargs):
     if len(args) == len(self._meta.concrete_fields):
         super(FeatureSet, self).__init__(*args, **kwargs)
@@ -55,10 +74,7 @@ def __init__(self, *args, **kwargs):
     # now we proceed with the user-facing constructor
     if len(args) > 1:
         raise ValueError("Only one non-keyword arg allowed: features")
-    features: Iterable[ORM] = kwargs.pop("features") if len(args) == 0 else args[0]
-    ref_field: Optional[str] = (
-        kwargs.pop("ref_field") if "ref_field" in kwargs else "id"
-    )
+    features: Iterable[Registry] = kwargs.pop("features") if len(args) == 0 else args[0]
     type: Optional[Union[type, str]] = kwargs.pop("type") if "type" in kwargs else None
     modality: Optional[str] = kwargs.pop("modality") if "modality" in kwargs else None
     name: Optional[str] = kwargs.pop("name") if "name" in kwargs else None
@@ -66,11 +82,11 @@ def __init__(self, *args, **kwargs):
     hash: Optional[str] = kwargs.pop("hash") if "hash" in kwargs else None
     if len(kwargs) > 0:
         raise ValueError(
-            "Only features, ref_field, type, modality, name are valid keyword arguments"
+            "Only features, type, modality, name are valid keyword arguments"
         )
 
     # now code
-    features_orm = validate_features(features)
+    features_orm = sanity_check_features(features)
     if features_orm == Feature:
         type = None
     else:
@@ -80,7 +96,7 @@ def __init__(self, *args, **kwargs):
         features_hash = hash_set({feature.id for feature in features})
         feature_set = FeatureSet.filter(hash=features_hash).one_or_none()
         if feature_set is not None:
-            logger.info(f"Loaded {feature_set}")
+            logger.success(f"loaded: {feature_set}")
             init_self_from_db(self, feature_set)
             return None
         else:
@@ -108,7 +124,7 @@ def __init__(self, *args, **kwargs):
         type=type_str,
         n=n_features,
         modality=modality_record,
-        ref_field=f"{features_orm.__get_name_with_schema__()}.{ref_field}",
+        registry=features_orm.__get_name_with_schema__(),
         hash=hash,
     )
 
@@ -139,39 +155,48 @@ def from_values(
     name: Optional[str] = None,
     modality: Optional[str] = None,
     **kwargs,
-) -> "FeatureSet":
+) -> Optional["FeatureSet"]:
     """{}"""
     if not isinstance(field, Field):
-        raise TypeError("Argument `field` must be an ORM field, e.g., `Feature.name`")
+        raise TypeError(
+            "Argument `field` must be an Registry field, e.g., `Feature.name`"
+        )
     if len(values) == 0:
         raise ValueError("Provide a list of at least one value")
-    ORM = field.field.model
-    if isinstance(ORM, Feature):
+    registry = field.field.model
+    if isinstance(registry, Feature):
         raise ValueError("Please use from_df() instead of from_values()")
     iterable_idx = index_iterable(values)
     if not isinstance(iterable_idx[0], (str, int)):
         raise TypeError("values should be list-like of str or int")
-    features_hash = hash_set(set(iterable_idx))
+    from_bionty = registry.__module__.startswith("lnschema_bionty")
+    features = get_or_create_records(
+        iterable=iterable_idx,
+        field=field,
+        from_bionty=from_bionty,
+        **kwargs,
+    )
+    validated_features = get_validated_features(features, field)
+    validated_feature_ids = [feature.id for feature in validated_features]
+    features_hash = hash_set(set(validated_feature_ids))
     feature_set = FeatureSet.filter(hash=features_hash).one_or_none()
     if feature_set is not None:
-        logger.info(f"Loaded {feature_set}")
+        logger.success(f"loaded {feature_set}")
     else:
-        from_bionty = ORM.__module__.startswith("lnschema_bionty")
-        records = get_or_create_records(
-            iterable=iterable_idx,
-            field=field,
-            from_bionty=from_bionty,
-            **kwargs,
-        )
-        # type_str = type.__name__ if not isinstance(type, str) else type
-        feature_set = FeatureSet(
-            features=records,
-            hash=features_hash,
-            name=name,
-            modality=modality,
-            type=type,
-            ref_field=field.field.name,
-        )
+        if type is not None:
+            type_str = type.__name__ if not isinstance(type, str) else type
+        else:
+            type_str = None
+        if validated_features:
+            feature_set = FeatureSet(
+                features=validated_features,
+                hash=features_hash,
+                name=name,
+                modality=modality,
+                type=type_str,
+            )
+        else:
+            feature_set = None
     return feature_set
 
 
@@ -181,10 +206,16 @@ def from_df(
     cls,
     df: "pd.DataFrame",
     name: Optional[str] = None,
-) -> "FeatureSet":
+) -> Optional["FeatureSet"]:
     """{}"""
     features = Feature.from_df(df)
-    feature_set = FeatureSet(features, name=name)
+    validated_features = get_validated_features(features, Feature.name)
+    if validated_features:
+        feature_set = FeatureSet(validated_features, name=name)
+    else:
+        logger.warning("no validated features, skip creating feature set")
+        feature_set = None
+        # raise ValidationError("Dataframe columns contain no validated feature names")
     return feature_set