lamindb 0.45.0__py3-none-any.whl → 0.46a1__py3-none-any.whl

lamindb/__init__.py

@@ -13,12 +13,14 @@ The core schema entities are central to lamindb's API:
 .. autosummary::
    :toctree: .
 
+   Dataset
    File
    Transform
    Run
+   Feature
+   FeatureSet
    User
    Storage
-   FeatureSet
    Tag
    Project
 
@@ -28,7 +30,6 @@ Functional tools:
    :toctree: .
 
    track
-   parse
    view
    select
    save
@@ -54,23 +55,43 @@ Developer API:
 
 """
 
-__version__ = "0.45.0"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.46a1"  # denote a release candidate for 0.1.0 with 0.1rc1
+
+import os as _os
 
 import lamindb_setup as _lamindb_setup
 
 # prints warning of python versions
 from lamin_logger import py_version_warning as _py_version_warning
 from lamindb_setup import _check_instance_setup
+from lamindb_setup._check_instance_setup import _INSTANCE_NOT_SETUP_WARNING
 
-_py_version_warning("3.8", "3.10")
+_py_version_warning("3.8", "3.11")
 
+_TESTING = _lamindb_setup._TESTING
 _INSTANCE_SETUP = _check_instance_setup(from_lamindb=True)
 # allow the user to call setup
 from . import setup  # noqa
 
+
+class InstanceNotSetupError(Exception):
+    pass
+
+
+def __getattr__(name):
+    raise InstanceNotSetupError(
+        f"{_INSTANCE_NOT_SETUP_WARNING}If you used the CLI to init or load an instance,"
+        " please RESTART the python session (in a notebook, restart kernel)"
+    )
+
+
 # only import all other functionality if setup was successful
 if _INSTANCE_SETUP:
+    del InstanceNotSetupError
+    del __getattr__  # delete so that imports work out
     from lnschema_core import (  # noqa
+        Dataset,
+        Feature,
         FeatureSet,
         File,
         Project,
@@ -95,12 +116,12 @@ if _INSTANCE_SETUP:
         f" {__version__})"
     )
 
-    from . import _baseorm_methods  # noqa
-    from . import _featureset_methods  # noqa
-    from . import _file_methods  # noqa
-    from . import _transform_methods  # noqa
+    from . import _dataset  # noqa
+    from . import _feature_set  # noqa
+    from . import _file  # noqa
+    from . import _orm  # noqa
+    from . import _transform  # noqa
     from ._delete import delete  # noqa
-    from ._from_values import parse  # noqa
     from ._save import save  # noqa
     from ._select import select  # noqa
     from ._view import view  # noqa

lamindb/_context.py

@@ -20,10 +20,9 @@ msg_path_failed = (
     " `notebook_path` to ln.track()."
 )
 
-msg_init_noninteractive = (
-    "Please attach an ID to the notebook by running the CLI: lamin track"
-    " my-notebook.ipynb"
-)
+
+class NonInteractiveEditorError(Exception):
+    pass
 
 
 def _write_notebook_meta(metadata):
@@ -178,10 +177,11 @@ class context:
                             " notebook!\nConsider installing nbproject for automatic"
                             " name, title & id tracking."
                         )
-                    elif str(e) == msg_init_noninteractive:
+                    elif isinstance(e, NonInteractiveEditorError):
                         raise e
                     else:
                         logger.warning(f"Automatic tracking of notebook failed: {e}")
+                        raise e
                     is_tracked_notebook = False
 
             if not is_tracked_notebook:
@@ -292,7 +292,7 @@ class context:
             except Exception as e:
                 nbproject_failed_msg = (
                     "Auto-retrieval of notebook name & title failed.\n\nFixes: Either"
-                    " init on the CLI `lamin track my-notebook.ipynb` or pass"
+                    f" init on the CLI `lamin track {notebook_path}` or pass"
                     " transform manually `ln.track(ln.Transform(name='My"
                     " notebook'))`\n\nPlease consider pasting error at:"
                     f" https://github.com/laminlabs/nbproject/issues/new\n\n{e}"
@@ -317,12 +317,11 @@ class context:
             if _env in ("lab", "notebook"):
                 cls._notebook_meta = metadata  # type: ignore
             else:
-                # nb = nbproject.dev.read_notebook(_filepath)
-                # nb.metadata["nbproject"] = metadata
-                # nbproject.dev.write_notebook(nb, _filepath)
-                # raise SystemExit(msg_init_complete)
-                # the following is safer
-                raise RuntimeError(msg_init_noninteractive)
+                msg_init_noninteractive = (
+                    "Please attach metadata to the notebook by running the CLI: "
+                    f"lamin track {notebook_path}"
+                )
+                raise NonInteractiveEditorError(msg_init_noninteractive)
 
         if _env in ("lab", "notebook"):
             # save the notebook in case that title was updated

lamindb/_dataset.py

@@ -0,0 +1,142 @@
+from typing import Iterable, List, Optional, Union
+
+import anndata as ad
+import pandas as pd
+from lnschema_core import ids
+from lnschema_core.models import Dataset
+
+from . import Feature, FeatureSet, File, Run
+from .dev.hashing import hash_set
+
+
+def __init__(
+    dataset: Dataset,
+    *args,
+    **kwargs,
+):
+    if len(args) == len(dataset._meta.concrete_fields):
+        super(Dataset, dataset).__init__(*args, **kwargs)
+        return None
+    # now we proceed with the user-facing constructor
+    if len(args) > 1:
+        raise ValueError("Only one non-keyword arg allowed: data")
+    data: Optional[Union[pd.DataFrame, ad.AnnData]] = None
+    if "data" in kwargs or len(args) == 1:
+        data = kwargs.pop("data") if len(args) == 0 else args[0]
+    name: Optional[str] = kwargs.pop("name") if "name" in kwargs else None
+    run: Optional[Run] = kwargs.pop("run") if "run" in kwargs else None
+    files: List[File] = kwargs.pop("files") if "files" in kwargs else []
+    file: Optional[File] = kwargs.pop("file") if "file" in kwargs else None
+    hash: Optional[str] = kwargs.pop("hash") if "hash" in kwargs else None
+    feature_sets: List[FeatureSet] = (
+        kwargs.pop("feature_sets") if "feature_sets" in kwargs else []
+    )
+    assert len(kwargs) == 0
+    if data is not None:
+        if isinstance(data, pd.DataFrame):
+            feature_set = FeatureSet.from_values(data.columns, Feature.name)
+            dataset._feature_sets = [feature_set]
+        elif isinstance(data, ad.AnnData):
+            if len(feature_sets) != 2:
+                raise ValueError(
+                    "Please provide a feature set describing each `.var.index` &"
+                    " `.obs.columns`"
+                )
+            dataset._feature_sets = feature_sets
+        file = File(data=data, run=run, feature_sets=dataset._feature_sets)
+        hash = file.hash
+        id = file.id
+    else:
+        id = ids.base62_20()
+        dataset._feature_sets = feature_sets
+    super(Dataset, dataset).__init__(id=id, name=name, file=file, hash=hash)
+    dataset._files = files
+
+
+@classmethod  # type: ignore
+def from_files(dataset: Dataset, *, name: str, files: Iterable[File]) -> Dataset:
+    # assert all files are already saved
+    # saved = not any([file._state._adding for file in files])
+    # if not saved:
+    #     raise ValueError("Not all files are yet saved, please save them")
+    # query all feature sets of files
+    file_ids = [file.id for file in files]
+    # query all feature sets at the same time rather than making a single query per file
+    feature_set_file_links = File.feature_sets.through.objects.filter(
+        file_id__in=file_ids
+    )
+    feature_set_ids = [link.featureset_id for link in feature_set_file_links]
+    feature_sets = FeatureSet.select(id__in=feature_set_ids)
+    # validate consistency of feature_sets
+    # we only allow one feature set per type
+    feature_set_types = [feature_set.type for feature_set in feature_sets]
+    feature_set_ids_types = [
+        (feature_set.id, feature_set.type) for feature_set in feature_sets
+    ]
+    if len(set(feature_set_ids_types)) != len(set(feature_set_types)):
+        # we can do below in the future!
+        # logger.warning(
+        #     "feature sets are inconsistent across files"
+        #     "computing union! files will be outer-joined"
+        # )
+        raise ValueError(
+            "Currently only supporting datasets from files with same feature sets"
+        )
+    # validate consistency of hashes
+    # we do not allow duplicate hashes
+    file_hashes = [file.hash for file in files]
+    file_hashes_set = set(file_hashes)
+    assert len(file_hashes) == len(file_hashes_set)
+    hash = hash_set(file_hashes_set)
+    # create the dataset
+    dataset = Dataset(name=name, hash=hash, feature_sets=feature_sets, files=files)
+    return dataset
+
+
+def backed(dataset: Dataset):
+    if dataset.file is None:
+        raise RuntimeError("Can only call backed() for datasets with a single file")
+    return dataset.file.backed()
+
+
+def load(dataset: Dataset):
+    """Load the combined dataset."""
+    if dataset.file is not None:
+        return dataset.file.load()
+    else:
+        suffixes = [file.suffix for file in dataset.files.all()]
+        if len(set(suffixes)) != 1:
+            raise RuntimeError(
+                "Can only load datasets where all files have the same suffix"
+            )
+        objects = [file.load() for file in dataset.files.all()]
+        if isinstance(objects[0], pd.DataFrame):
+            return pd.concat(objects)
+        elif isinstance(objects[0], ad.AnnData):
+            return ad.concat(objects)
+
+
+def delete(dataset: Dataset, storage: bool = False):
+    super(Dataset, dataset).delete()
+    if dataset.file is not None:
+        dataset.file.delete(storage=storage)
+
+
+def save(dataset: Dataset):
+    if dataset.file is not None:
+        dataset.file.save()
+    for feature_set in dataset._feature_sets:
+        feature_set.save()
+    super(Dataset, dataset).save()
+    if len(dataset._files) > 0:
+        dataset.files.set(dataset._files)
+    if len(dataset._feature_sets) > 0:
+        dataset.feature_sets.set(dataset._feature_sets)
+
+
+Dataset.__init__ = __init__
+Dataset.from_files = from_files
+Dataset.backed = backed
+Dataset.load = load
+Dataset.delete = delete
+Dataset.save = save

lamindb/_delete.py

@@ -1,31 +1,31 @@
 from typing import List, Optional, Union, overload  # noqa
 
-from lnschema_core import BaseORM
+from lnschema_core import ORM
 
 from ._logger import colors, logger
 
 
 @overload
 def delete(
-    record: BaseORM,
+    record: ORM,
 ) -> None:
     ...
 
 
 @overload
 def delete(
-    records: List[BaseORM],
+    records: List[ORM],
 ) -> None:  # type: ignore
     ...
 
 
 def delete(  # type: ignore
-    records: Union[BaseORM, List[BaseORM]],
+    records: Union[ORM, List[ORM]],
 ) -> None:
     """Delete metadata records & files.
 
     Args:
-        records: `Union[BaseORM, List[BaseORM]]` One or multiple records.
+        records: `Union[ORM, List[ORM]]` One or multiple records.
 
     Returns:
         `None`
@@ -49,7 +49,7 @@ def delete(  # type: ignore
     """
     if isinstance(records, list):
         records = records
-    elif isinstance(records, BaseORM):
+    elif isinstance(records, ORM):
         records = [records]
     for record in records:
         record.delete()

lamindb/_feature_set.py

@@ -0,0 +1,138 @@
+from typing import List, Optional
+
+from django.db.models.query_utils import DeferredAttribute as Field
+from lamin_logger import logger
+from lamindb_setup.dev._docs import doc_args
+from lnschema_core import ORM, Feature, FeatureSet
+from lnschema_core.types import ListLike
+
+from lamindb.dev.hashing import hash_set
+from lamindb.dev.utils import attach_func_to_class_method
+
+from . import _TESTING
+from ._from_values import get_or_create_records, index_iterable
+from ._orm import init_self_from_db
+
+
+def get_related_name(features_type: ORM):
+    candidates = [
+        field.related_name
+        for field in FeatureSet._meta.related_objects
+        if field.related_model == features_type
+    ]
+    if not candidates:
+        raise ValueError(
+            f"Can't create feature sets from {features_type.__name__} because it's not"
+            " related to it!\nYou need to create a link model between FeatureSet and"
+            " your ORM in your custom schema.\nTo do so, add a line:\nfeature_sets ="
+            " models.ManyToMany(FeatureSet, related_name='mythings')\n"
+        )
+    return candidates[0]
+
+
+def validate_features(features: List[ORM]) -> ORM:
+    """Validate and return feature type."""
+    if len(features) == 0:
+        raise ValueError("provide list of features with at least one element")
+    if not hasattr(features, "__getitem__"):
+        raise TypeError("features has to be list-like")
+    if not isinstance(features[0], ORM):
+        raise TypeError(
+            "features has to store feature records! use .from_values() otherwise"
+        )
+    feature_types = set([feature.__class__ for feature in features])
+    if len(feature_types) > 1:
+        raise ValueError("feature_set can only contain a single type")
+    return next(iter(feature_types))  # return value in set of cardinality 1
+
+
+def __init__(self, *args, **kwargs):
+    if len(args) == len(self._meta.concrete_fields):
+        super(FeatureSet, self).__init__(*args, **kwargs)
+        return None
+    # now we proceed with the user-facing constructor
+    if len(args) > 1:
+        raise ValueError("Only one non-keyword arg allowed: features")
+    features: List[ORM] = kwargs.pop("features") if len(args) == 0 else args[0]
+    field: Optional[str] = kwargs.pop("field") if "field" in kwargs else None
+    id: Optional[str] = kwargs.pop("id") if "id" in kwargs else None
+    features_type = validate_features(features)
+    related_name = get_related_name(features_type)
+    if id is None:
+        features_hash = hash_set({feature.id for feature in features})
+        feature_set = FeatureSet.select(id=features_hash).one_or_none()
+        if feature_set is not None:
+            logger.info("Returning an existing feature_set")
+            init_self_from_db(self, feature_set)
+            return None
+        else:
+            id = features_hash
+    self._features = (related_name, features)
+    if field is None:
+        field = "id"
+    super(FeatureSet, self).__init__(
+        id=id, type=features_type.__name_with_type__(), field=field
+    )
+
+
+@doc_args(FeatureSet.save.__doc__)
+def save(self, *args, **kwargs) -> None:
+    """{}"""
+    super(FeatureSet, self).save(*args, **kwargs)
+    if hasattr(self, "_features"):
+        related_name, records = self._features
+        [record.save() for record in records]
+        getattr(self, related_name).set(records)
+
+
+@classmethod  # type:ignore
+@doc_args(FeatureSet.from_values.__doc__)
+def from_values(
+    cls, values: ListLike, field: Field = Feature.name, **kwargs
+) -> "FeatureSet":
+    """{}"""
+    if not isinstance(field, Field):
+        raise TypeError("Argument `field` must be an ORM field, e.g., `Feature.name`")
+    if len(values) == 0:
+        raise ValueError("Provide a list of at least one value")
+    if not isinstance(values[0], (str, int)):
+        raise TypeError("values should be list-like of str or int")
+    orm = field.field.model
+    iterable_idx = index_iterable(values)
+    features_hash = hash_set(set(iterable_idx))
+    feature_set = FeatureSet.select(id=features_hash).one_or_none()
+    if feature_set is not None:
+        logger.info("Returning an existing feature_set")
+    else:
+        from_bionty = orm.__module__.startswith("lnschema_bionty")
+        records = get_or_create_records(
+            iterable=iterable_idx,
+            field=field,
+            from_bionty=from_bionty,
+            **kwargs,
+        )
+        feature_set = FeatureSet(
+            id=features_hash,
+            field=field.field.name,
+            features=records,
+        )
+    return feature_set
+
+
+METHOD_NAMES = [
+    "__init__",
+    "from_values",
+    "save",
+]
+
+if _TESTING:
+    from inspect import signature
+
+    SIGS = {
+        name: signature(getattr(FeatureSet, name))
+        for name in METHOD_NAMES
+        if name != "__init__"
+    }
+
+for name in METHOD_NAMES:
+    attach_func_to_class_method(name, FeatureSet, globals())