lamindb 0.48a3__py3-none-any.whl → 0.48.1__py3-none-any.whl

lamindb/_from_values.py

@@ -2,6 +2,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
 
 import pandas as pd
 from django.core.exceptions import FieldDoesNotExist
+from django.db.models import Case, When
 from django.db.models.query_utils import DeferredAttribute as Field
 from lamin_utils import colors, logger
 from lnschema_core.models import ORM, Feature
@@ -17,7 +18,7 @@ def get_or_create_records(
     *,
     from_bionty: bool = False,
     **kwargs,
-) -> List:
+) -> List[ORM]:
     """Get or create records from iterables."""
     upon_create_search_names = settings.upon_create_search_names
     settings.upon_create_search_names = False
@@ -59,7 +60,7 @@ def get_or_create_records(
                         params["type"] = str(types[value])
                     records.append(ORM(**params, **kwargs))
                 s = "" if len(unmapped_values) == 1 else "s"
-                print_unmapped_values = ", ".join(unmapped_values[:7])
+                print_unmapped_values = ", ".join(unmapped_values[:10])
                 if len(unmapped_values) > 10:
                     print_unmapped_values += ", ..."
                 additional_info = " "
@@ -69,6 +70,27 @@ def get_or_create_records(
                     f"Created {colors.yellow(f'{len(unmapped_values)} {ORM.__name__} record{s}')} for{additional_info}"  # noqa
                     f"{colors.yellow(f'{field_name}{s}')}: {print_unmapped_values}"  # noqa
                 )
+        if ORM.__module__.startswith("lnschema_bionty."):
+            if isinstance(iterable, pd.Series):
+                feature = iterable.name
+            else:
+                logger.warning(
+                    "Did not receive values as pd.Series, inferring feature from"
+                    f" reference ORM: {ORM.__name__}"
+                )
+                feature = ORM.__name__.lower()
+            if isinstance(feature, str):
+                feature_name = feature
+                feature = Feature.select(name=feature).one_or_none()
+            elif feature is not None:
+                feature_name = feature.name
+            if feature is not None:
+                for record in records:
+                    record._feature = feature
+            if feature_name is not None:
+                for record in records:
+                    record._feature = feature_name
+            logger.info(f"Mapping records to feature '{feature_name}'")
         return records
     finally:
         settings.upon_create_search_names = upon_create_search_names
@@ -97,10 +119,14 @@ def get_existing_records(iterable_idx: pd.Index, field: Field, kwargs: Dict = {}
     syn_msg = ""
     if len(syn_mapper) > 0:
         s = "" if len(syn_mapper) == 1 else "s"
+        names = list(syn_mapper.keys())
+        print_values = ", ".join(names[:10])
+        if len(names) > 10:
+            print_values += ", ..."
         syn_msg = (
             "Loaded"
             f" {colors.green(f'{len(syn_mapper)} {model.__name__} record{s}')} that"  # noqa
-            f" matched {colors.green('synonyms')}"
+            f" matched {colors.green('synonyms')}: {print_values}"
         )
         iterable_idx = iterable_idx.to_frame().rename(index=syn_mapper).index
 
@@ -112,22 +138,37 @@ def get_existing_records(iterable_idx: pd.Index, field: Field, kwargs: Dict = {}
 
     from ._select import select
 
-    stmt = select(model, **condition)
+    query_set = select(model, **condition)
+
+    # new we have to sort the list of queried records
+    preserved = Case(
+        *[
+            When(**{field_name: value}, then=pos)
+            for pos, value in enumerate(iterable_idx)
+        ]
+    )
+    records = query_set.order_by(preserved).list()
 
-    records = stmt.list()  # existing records
     n_name = len(records) - len(syn_mapper)
+    names = [getattr(record, field_name) for record in records]
+    names = [name for name in names if name not in syn_mapper.values()]
     if n_name > 0:
         s = "" if n_name == 1 else "s"
+        print_values = ", ".join(names[:10])
+        if len(names) > 10:
+            print_values += ", ..."
         logger.info(
             "Loaded"
             f" {colors.green(f'{n_name} {model.__name__} record{s}')} that"
-            f" matched {colors.green(f'{field_name}')}"
+            f" matched {colors.green(f'{field_name}')}: {print_values}"
         )
     # make sure that synonyms logging appears after the field logging
     if len(syn_msg) > 0:
         logger.info(syn_msg)
 
-    existing_values = iterable_idx.intersection(stmt.values_list(field_name, flat=True))
+    existing_values = iterable_idx.intersection(
+        query_set.values_list(field_name, flat=True)
+    )
     nonexist_values = iterable_idx.difference(existing_values)
 
     return records, nonexist_values
@@ -161,10 +202,14 @@ def create_records_from_bionty(
     msg_syn: str = ""
     if len(syn_mapper) > 0:
         s = "" if len(syn_mapper) == 1 else "s"
+        names = list(syn_mapper.keys())
+        print_values = ", ".join(names[:10])
+        if len(names) > 10:
+            print_values += ", ..."
         msg_syn = (
             "Loaded"
             f" {colors.purple(f'{len(syn_mapper)} {model.__name__} record{s} from Bionty')} that"  # noqa
-            f" matched {colors.purple('synonyms')}"
+            f" matched {colors.purple('synonyms')}: {print_values}"
         )
 
         iterable_idx = iterable_idx.to_frame().rename(index=syn_mapper).index
@@ -179,26 +224,24 @@ def create_records_from_bionty(
         for bk in bionty_kwargs:
             records.append(model(**bk, **kwargs))
 
-        # logging of BiontySource linking
-        source_msg = (
-            ""
-            if kwargs.get("bionty_source") is None
-            else f" (bionty_source_id={kwargs.get('bionty_source').id})"  # type:ignore # noqa
-        )
-
         # number of records that matches field (not synonyms)
         n_name = len(records) - len(syn_mapper)
+        names = [getattr(record, field_name) for record in records]
+        names = [name for name in names if name not in syn_mapper.values()]
         if n_name > 0:
             s = "" if n_name == 1 else "s"
+            print_values = ", ".join(names[:10])
+            if len(names) > 10:
+                print_values += ", ..."
             msg = (
                 "Loaded"
                 f" {colors.purple(f'{n_name} {model.__name__} record{s} from Bionty')} that"  # noqa
-                f" matched {colors.purple(f'{field_name}')}"
+                f" matched {colors.purple(f'{field_name}')}: {print_values}"
             )
-            logger.info(msg + source_msg)
+            logger.info(msg)
         # make sure that synonyms logging appears after the field logging
         if len(msg_syn) > 0:
-            logger.info(msg_syn + source_msg)
+            logger.info(msg_syn)
         # warning about multi matches
         if len(multi_msg) > 0:
             logger.warning(multi_msg)

lamindb/_label.py

@@ -1,6 +1,7 @@
 from typing import List, Optional, Union
 
 import pandas as pd
+from lamin_utils import logger
 from lamindb_setup.dev._docs import doc_args
 from lnschema_core import Feature, Label
 from lnschema_core.types import ListLike
@@ -11,6 +12,40 @@ from . import _TESTING
 from ._from_values import get_or_create_records, index_iterable
 
 
+def __init__(self, *args, **kwargs):
+    if len(args) == len(self._meta.concrete_fields):
+        super(Label, self).__init__(*args, **kwargs)
+        return None
+    # now we proceed with the user-facing constructor
+    if len(args) > 0:
+        raise ValueError("Only one non-keyword arg allowed")
+    name: Optional[str] = kwargs.pop("name") if "name" in kwargs else None
+    description: Optional[str] = (
+        kwargs.pop("description") if "description" in kwargs else None
+    )
+    feature: Optional[str] = kwargs.pop("feature") if "feature" in kwargs else None
+    feature_id: Optional[str] = (
+        kwargs.pop("feature_id") if "feature_id" in kwargs else None
+    )
+    if len(kwargs) > 0:
+        raise ValueError("Only name, description, feature are valid keyword arguments")
+    # continue
+    if feature is None and feature_id is None:
+        logger.warning("Consider passing a corresponding feature for your label!")
+    if isinstance(feature, str):
+        feature = Feature.select(name=feature).one_or_none()
+        if feature is None:
+            raise ValueError(
+                f"Feature with name {feature} does not exist, please create it:"
+                f" ln.Feature(name={feature}, type='float')"
+            )
+        else:
+            feature_id = feature.id
+    super(Label, self).__init__(
+        name=name, description=description, feature_id=feature_id
+    )
+
+
 @classmethod  # type:ignore
 @doc_args(Label.from_values.__doc__)
 def from_values(
@@ -33,6 +68,7 @@ def from_values(
 
 
 METHOD_NAMES = [
+    "__init__",
     "from_values",
 ]
 

lamindb/_manager.py

@@ -43,9 +43,9 @@ class Manager(models.Manager):
         else:
             return [item for item in self.values_list(field, flat=True)]
 
-    def df(self):
+    def df(self, **kwargs):
         """Convert to DataFrame."""
-        return self.all().df()
+        return self.all().df(**kwargs)
 
 
 setattr(models.Manager, "list", Manager.list)

lamindb/_orm.py

@@ -5,7 +5,7 @@ import pandas as pd
 from django.core.exceptions import FieldDoesNotExist
 from django.db.models import Manager, QuerySet
 from django.db.models.query_utils import DeferredAttribute as Field
-from lamin_utils import logger
+from lamin_utils import colors, logger
 from lamin_utils._lookup import Lookup
 from lamin_utils._search import search as base_search
 from lamindb_setup.dev._docs import doc_args
@@ -16,7 +16,9 @@ from lnschema_core.types import ListLike, StrField
 from lamindb.dev.utils import attach_func_to_class_method
 
 from . import _TESTING
+from ._feature_manager import create_features_df
 from ._from_values import _has_species_field, get_or_create_records
+from .dev._settings import settings
 
 IPYTHON = getattr(builtins, "__IPYTHON__", False)
 
@@ -395,9 +397,40 @@ def map_synonyms(
     )
 
 
+def _labels_with_feature_names(labels: Union[QuerySet, Manager]) -> Dict:
+    from django.db.models import F
+
+    df = labels.annotate(feature_name=F("feature__name")).df()
+    return df.groupby("feature_name")["name"].apply(list).to_dict()
+
+
 def describe(self):
-    model_name = self.__class__.__name__
+    model_name = colors.green(self.__class__.__name__)
     msg = ""
+
+    def dict_related_model_to_related_name(orm):
+        d: Dict = {
+            f"{i.related_model.__get_schema_name__()}.{i.related_model.__name__}": (
+                i.related_name
+            )
+            for i in orm._meta.related_objects
+            if i.related_name is not None
+        }
+        d.update(
+            {
+                f"{i.related_model.__get_schema_name__()}.{i.related_model.__name__}": (
+                    i.name
+                )
+                for i in orm._meta.many_to_many
+                if i.name is not None
+            }
+        )
+
+        return d
+
+    file_related_models = dict_related_model_to_related_name(self)
+
+    # Display the file record
     fields = self._meta.fields
     direct_fields = []
     foreign_key_fields = []
@@ -406,42 +439,126 @@ def describe(self):
             foreign_key_fields.append(f.name)
         else:
             direct_fields.append(f.name)
+
+    # Display Provenance
     # display line by line the foreign key fields
+    emojis = {"storage": "💾", "created_by": "👤", "transform": "💫", "run": "🚗"}
     if len(foreign_key_fields) > 0:
         record_msg = f"{model_name}({''.join([f'{i}={self.__getattribute__(i)}, ' for i in direct_fields])})"  # noqa
         msg += f"{record_msg.rstrip(', )')})\n\n"
 
-        msg += "One/Many-to-One:\n    "
+        msg += f"{colors.green('Provenance')}:\n    "
         related_msg = "".join(
-            [f"🔗 {i}: {self.__getattribute__(i)}\n    " for i in foreign_key_fields]
+            [
+                f"{emojis.get(i, '📎')} {i}: {self.__getattribute__(i)}\n    "
+                for i in foreign_key_fields
+            ]
         )
         msg += related_msg
+    # input of
+    if self.input_of.exists():
+        values = [format_datetime(i.run_at) for i in self.input_of.all()]
+        msg += f"⬇️ input_of ({colors.italic('core.Run')}): {values}\n    "
     msg = msg.rstrip("    ")
 
-    # display many-to-many relationship objects
-    # fields in the model definition
-    related_names = [i.name for i in self._meta.many_to_many]
-    # fields back linked
-    related_names += [i.related_name for i in self._meta.related_objects]
-    msg += "Many-to-Many:\n"
-    for related_name in related_names:
-        related_objects = self.__getattribute__(related_name)
-        count = related_objects.count()
-        if count > 0:
-            try:
-                field = get_default_str_field(related_objects)
-            except ValueError:
-                field = "id"
-            objects_list = list(related_objects.values_list(field, flat=True)[:10])
-            if field == "created_at":
-                objects_list = [format_datetime(i) for i in objects_list]
-            msg_objects = f"    🔗 {related_name} ({count}): {objects_list}\n"
-            if count > 10:
-                msg_objects = msg_objects.replace("]", " ... ]")
-            msg += msg_objects
+    if not self.feature_sets.exists():
+        print(msg)
+        return
+    else:
+        feature_sets_related_models = dict_related_model_to_related_name(
+            self.feature_sets.first()
+        )
+    # Display Features by slot
+    msg += f"{colors.green('Features')}:\n"
+    # var
+    feature_sets = self.feature_sets.exclude(ref_orm="Feature")
+    if feature_sets.exists():
+        for feature_set in feature_sets.all():
+            key = f"{feature_set.ref_schema}.{feature_set.ref_orm}"
+            related_name = feature_sets_related_models.get(key)
+            values = (
+                feature_set.__getattribute__(related_name)
+                .all()[:5]
+                .list(feature_set.ref_field)
+            )
+            slots = self.feature_sets.through.objects.filter(
+                file=self, feature_set=feature_set
+            ).list("slot")
+            for slot in slots:
+                if slot == "var":
+                    slot += " (X)"
+                msg += f"  🗺️ {colors.bold(slot)}:\n"
+                ref = colors.italic(f"{key}.{feature_set.ref_field}")
+                msg += f"    🔗 index ({feature_set.n}, {ref}): {values}\n".replace(
+                    "]", "...]"
+                )
+
+    # obs
+    # ref_orm=Feature, combine all features into one dataframe
+    feature_sets = self.feature_sets.filter(ref_orm="Feature").all()
+    if feature_sets.exists():
+        features_df = create_features_df(
+            file=self, feature_sets=feature_sets.all(), exclude=True
+        )
+        for slot in features_df["slot"].unique():
+            df_slot = features_df[features_df.slot == slot]
+            if slot == "obs":
+                slot += " (metadata)"
+            msg += f"  🗺️ {colors.bold(slot)}:\n"
+            df_label_index = df_slot[
+                (df_slot["labels_orm"] == "Label")
+                & (df_slot["labels_schema"] == "core")
+            ].index
+
+            # for labels
+            if len(df_label_index) > 0:
+                labels_schema = "core"
+                labels_orm = "Label"
+                key = f"{labels_schema}.{labels_orm}"
+                related_name = file_related_models.get(key)
+                related_objects = self.__getattribute__(related_name)
+                labels = _labels_with_feature_names(related_objects)
+                msg_objects = ""
+                for k, v in labels.items():
+                    msg_objects_k = (
+                        f"    🔗 {k} ({len(v)}, {colors.italic(key)}): {v[:5]}\n"
+                    )
+                    if len(v) > 5:
+                        msg_objects_k = msg_objects_k.replace("]", " ... ]")
+                    msg_objects += msg_objects_k
+                msg += msg_objects
+
+            # for non-labels
+            nonlabel_index = df_slot.index.difference(df_label_index)
+            if len(nonlabel_index) == 0:
+                continue
+            df_nonlabels = df_slot.loc[nonlabel_index]
+            df_nonlabels = (
+                df_nonlabels.groupby(["labels_schema", "labels_orm"], group_keys=False)[
+                    "name"
+                ]
+                .apply(lambda x: "|".join(x))
+                .reset_index()
+            )
+            for _, row in df_nonlabels.iterrows():
+                key = f"{row.labels_schema}.{row.labels_orm}"
+                related_name = file_related_models.get(key)
+                related_objects = self.__getattribute__(related_name)
+                count = related_objects.count()
+                count_str = f"{count}, {colors.italic(f'{key}')}"
+                try:
+                    field = get_default_str_field(related_objects)
+                except ValueError:
+                    field = "id"
+                values = list(related_objects.values_list(field, flat=True)[:5])
+                msg_objects = f"    🔗 {row['name']} ({count_str}): {values}\n"
+                msg += msg_objects
     msg = msg.rstrip("\n")
-    msg = msg.rstrip("Many-to-Many:")
-    print(msg)
+    msg = msg.rstrip("Features:")
+    verbosity = settings.verbosity
+    settings.verbosity = 2
+    logger.info(msg)
+    settings.verbosity = verbosity
 
 
 def set_abbr(self, value: str):

lamindb/_queryset.py

@@ -100,6 +100,8 @@ class QuerySet(models.QuerySet):
             df.run_at = format_and_convert_to_local_time(df.run_at)
         if "id" in df.columns:
             df = df.set_index("id")
+        if len(df) == 0:
+            return df
         if include is not None:
             if isinstance(include, str):
                 include = [include]
@@ -121,7 +123,7 @@ class QuerySet(models.QuerySet):
                     if field.field.model != ORM
                     else field.field.related_model
                 )
-                if field.field.model == related_ORM:
+                if ORM == related_ORM:
                     left_side_link_model = f"from_{ORM.__name__.lower()}"
                     values_expression = f"to_{ORM.__name__.lower()}__{lookup_str}"
                 else:
@@ -135,7 +137,7 @@ class QuerySet(models.QuerySet):
                 link_groupby = link_df.groupby(left_side_link_model)[
                     values_expression
                 ].apply(list)
-                df = pd.concat((link_groupby, df), axis=1)
+                df = pd.concat((link_groupby, df), axis=1, join="inner")
                 df.rename(columns={values_expression: expression}, inplace=True)
         return df
 

lamindb/_save.py

@@ -76,13 +76,23 @@ def save(records: Iterable[ORM], **kwargs) -> None:  # type: ignore
         non_files_with_parents = {r for r in non_files if hasattr(r, "_parents")}
 
         if len(non_files_with_parents) > 0 and kwargs.get("parents") is not False:
-            # save the record with parents one by one
-            logger.warning(
-                "Now recursing through parents: "
-                "this only happens once, but is much slower than bulk saving"
-            )
-            for record in non_files_with_parents:
-                record._save_ontology_parents()
+            # this can only happen within lnschema_bionty right now!!
+            # we might extend to core lamindb later
+            import lnschema_bionty as lb
+
+            if kwargs.get("parents") or (
+                kwargs.get("parents") is None and lb.settings.auto_save_parents
+            ):
+                # save the record with parents one by one
+                logger.warning(
+                    "Now recursing through parents: "
+                    "this only happens once, but is much slower than bulk saving"
+                )
+                logger.hint(
+                    "You can switch this off via: lb.settings.auto_save_parents = False"
+                )
+                for record in non_files_with_parents:
+                    record._save_ontology_parents()
 
     if files:
         with transaction.atomic():

lamindb/dev/__init__.py

@@ -6,16 +6,20 @@
    ORM
    QuerySet
    Manager
+   FeatureManager
    datasets
    hashing
    storage
    Settings
+   run_context
 """
 
 from lnschema_core.models import ORM
 
+from lamindb._feature_manager import FeatureManager
 from lamindb._manager import Manager
 from lamindb._queryset import QuerySet
 
+from .._context import run_context
 from . import datasets  # noqa
 from ._settings import Settings