lamindb 0.71.2__py3-none-any.whl → 0.72.0__py3-none-any.whl

lamindb/__init__.py

@@ -41,7 +41,7 @@ Modules & settings:
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.71.2"
+__version__ = "0.72.0"
 
 import os as _os
 
@@ -72,6 +72,7 @@ if _check_instance_setup(from_lamindb=True):
         User,
     )
 
+    from . import core  # isort: split
     from . import (
         _annotate,
         _artifact,
@@ -86,7 +87,6 @@ if _check_instance_setup(from_lamindb=True):
         _storage,
         _transform,
         _ulabel,
-        core,
     )
 
     dev = core  # backward compat

lamindb/_annotate.py

@@ -9,18 +9,14 @@ from lamin_utils import colors, logger
 from lamindb_setup.core._docs import doc_args
 from lnschema_core import Artifact, Collection, Feature, Registry, Run, ULabel
 
+from .core.exceptions import ValidationError
+
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
     from lnschema_core.types import FieldAttr
     from mudata import MuData
 
 
-class ValidationError(ValueError):
-    """Validation error."""
-
-    pass
-
-
 class AnnotateLookup:
     """Lookup categories from the reference instance."""
 
@@ -566,7 +562,7 @@ class MuDataAnnotator:
             save_function="add_new_from_var_index",
             using=self._using,
             validated_only=validated_only,
-            type="number",
+            dtype="number",
             **kwargs,
         )
 
@@ -1034,7 +1030,7 @@ def update_registry(
     validated_only: bool = True,
     df: pd.DataFrame | None = None,
     organism: str | None = None,
-    type: str | None = None,
+    dtype: str | None = None,
     **kwargs,
 ) -> None:
     """Save features or labels records in the default instance from the using instance.
@@ -1048,7 +1044,7 @@ def update_registry(
         validated_only: If True, only save validated labels.
         df: A DataFrame to save labels from.
         organism: The organism name.
-        type: The type of the feature.
+        dtype: The type of the feature.
         kwargs: Additional keyword arguments to pass to the registry model to create new records.
     """
     from lamindb._save import save as ln_save
@@ -1102,7 +1098,7 @@ def update_registry(
                 for value in labels_saved["without reference"]:
                     filter_kwargs[field.field.name] = value
                     if registry == Feature:
-                        filter_kwargs["type"] = "category" if type is None else type
+                        filter_kwargs["dtype"] = "cat" if dtype is None else dtype
                     non_validated_records.append(registry(**filter_kwargs, **kwargs))
             ln_save(non_validated_records)
 

lamindb/_artifact.py

@@ -1,12 +1,15 @@
 from __future__ import annotations
 
+import os
 import shutil
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path, PurePath, PurePosixPath
 from typing import TYPE_CHECKING, Any, Mapping
 
 import fsspec
 import lamindb_setup as ln_setup
 import pandas as pd
+import psutil
 from anndata import AnnData
 from lamin_utils import colors, logger
 from lamindb_setup import settings as setup_settings
@@ -204,15 +207,26 @@ def get_stat_or_artifact(
             return size, hash, hash_type, n_objects
     else:
         if path.is_dir():
-            md5s = []
-            size = 0
-            for subpath in path.rglob("*"):
-                if not subpath.is_file():
-                    continue
-                size += subpath.stat().st_size
-                md5s.append(hash_file(subpath)[0])
-            hash, hash_type = hash_md5s_from_dir(md5s)
-            n_objects = len(md5s)
+            files = (subpath for subpath in path.rglob("*") if subpath.is_file())
+
+            def hash_size(file):
+                file_size = file.stat().st_size
+                return hash_file(file, file_size)[0], file_size
+
+            try:
+                n_workers = len(psutil.Process().cpu_affinity())
+            except AttributeError:
+                n_workers = psutil.cpu_count()
+            if n_workers > 1:
+                with ThreadPoolExecutor(n_workers) as pool:
+                    hashes_sizes = pool.map(hash_size, files)
+            else:
+                hashes_sizes = map(hash_size, files)
+            hashes, sizes = zip(*hashes_sizes)
+
+            hash, hash_type = hash_md5s_from_dir(hashes)
+            n_objects = len(hashes)
+            size = sum(sizes)
         else:
             hash, hash_type = hash_file(path)
             size = stat.st_size
@@ -335,7 +349,7 @@ def get_artifact_kwargs_from_data(
             # save the information that this artifact was previously
             # produced by another run
             if artifact.run is not None:
-                artifact.run.replicated_output_artifacts.add(artifact)
+                artifact.run.output_artifacts_with_later_updates.add(artifact)
             # update the run of the artifact with the latest run
             stat_or_artifact.run = run
             stat_or_artifact.transform = run.transform

lamindb/_can_validate.py

@@ -80,7 +80,9 @@ def _inspect(
 
     # inspect in the DB
     result_db = inspect(
-        df=_filter_query_based_on_organism(queryset=queryset, organism=organism),
+        df=_filter_query_based_on_organism(
+            queryset=queryset, field=field, organism=organism
+        ),
         identifiers=values,
         field=field,
         mute=mute,
@@ -161,6 +163,7 @@ def _validate(
     field_values = pd.Series(
         _filter_query_based_on_organism(
             queryset=queryset,
+            field=field,
             organism=organism,
             values_list_field=field,
         ),
@@ -284,7 +287,9 @@ def _standardize(
 
     try:
         orm._meta.get_field(synonyms_field)
-        df = _filter_query_based_on_organism(queryset=queryset, organism=organism)
+        df = _filter_query_based_on_organism(
+            queryset=queryset, field=field, organism=organism
+        )
     except FieldDoesNotExist:
         df = pd.DataFrame()
 
@@ -439,6 +444,7 @@ def _check_synonyms_field_exist(record: Registry):
 
 def _filter_query_based_on_organism(
     queryset: QuerySet,
+    field: str,
     organism: str | Registry | None = None,
     values_list_field: str | None = None,
 ):
@@ -447,7 +453,7 @@ def _filter_query_based_on_organism(
 
     orm = queryset.model
 
-    if _has_organism_field(orm):
+    if _has_organism_field(orm) and not field.endswith("id"):
         # here, we can safely import lnschema_bionty
         from lnschema_bionty._bionty import create_or_get_organism_record
 

lamindb/_collection.py

@@ -103,9 +103,9 @@ def __init__(
             if meta._state.adding:
                 raise ValueError("Save meta artifact before creating collection!")
             if not feature_sets:
-                feature_sets = meta.features._feature_set_by_slot
+                feature_sets = meta.features.feature_set_by_slot
             else:
-                if len(meta.features._feature_set_by_slot) > 0:
+                if len(meta.features.feature_set_by_slot) > 0:
                     logger.info("overwriting feature sets linked to artifact")
     # we ignore collections in trash containing the same hash
     if hash is not None:
@@ -121,7 +121,7 @@ def __init__(
             # save the information that this artifact was previously
             # produced by another run
             if existing_collection.run is not None:
-                existing_collection.run.replicated_output_collections.add(
+                existing_collection.run.output_collections_with_later_updates.add(
                     existing_collection
                 )
             # update the run of the artifact with the latest run
@@ -129,7 +129,7 @@ def __init__(
             existing_collection.transform = run.transform
         init_self_from_db(collection, existing_collection)
         update_attributes(collection, {"description": description, "name": name})
-        for slot, feature_set in collection.features._feature_set_by_slot.items():
+        for slot, feature_set in collection.features.feature_set_by_slot.items():
             if slot in feature_sets:
                 if not feature_sets[slot] == feature_set:
                     collection.feature_sets.remove(feature_set)
@@ -177,7 +177,7 @@ def from_artifacts(artifacts: Iterable[Artifact]) -> tuple[str, dict[str, str]]:
     feature_sets_by_slots = defaultdict(list)
     logger.debug("slots")
     for link in feature_set_artifact_links:
-        feature_sets_by_slots[link.slot].append(link.feature_set_id)
+        feature_sets_by_slots[link.slot].append(link.featureset_id)
     feature_sets_union = {}
     logger.debug("union")
     for slot, feature_set_ids_slot in feature_sets_by_slots.items():
@@ -197,7 +197,7 @@ def from_artifacts(artifacts: Iterable[Artifact]) -> tuple[str, dict[str, str]]:
         )
         start_time = logger.debug("done, start evaluate", time=start_time)
         features = features_registry.filter(id__in=feature_ids)
-        feature_sets_union[slot] = FeatureSet(features, type=feature_set_1.type)
+        feature_sets_union[slot] = FeatureSet(features, dtype=feature_set_1.dtype)
         start_time = logger.debug("done", time=start_time)
     # validate consistency of hashes
     # we do not allow duplicate hashes
@@ -361,7 +361,7 @@ def restore(self) -> None:
 @doc_args(Collection.artifacts.__doc__)
 def artifacts(self) -> QuerySet:
     """{}."""
-    return self.unordered_artifacts.order_by("collectionartifact__id")
+    return self.unordered_artifacts.order_by("collection_links__id")
 
 
 METHOD_NAMES = [

lamindb/_feature.py

@@ -1,26 +1,29 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING
 
 import lamindb_setup as ln_setup
 import pandas as pd
 from lamindb_setup.core._docs import doc_args
-from lnschema_core.models import Feature, Registry
+from lnschema_core.models import Artifact, Feature
 from pandas.api.types import CategoricalDtype, is_string_dtype
 
 from lamindb._utils import attach_func_to_class_method
 from lamindb.core._settings import settings
 
 from ._query_set import RecordsList
+from .core.schema import dict_schema_name_to_model_name
 
 if TYPE_CHECKING:
     from lnschema_core.types import FieldAttr
 
 FEATURE_TYPES = {
-    "int": "number",
-    "float": "number",
-    "str": "category",
-    "object": "category",
+    "number": "number",
+    "int": "int",
+    "float": "float",
+    "bool": "bool",
+    "str": "cat",
+    "object": "cat",
 }
 
 
@@ -28,10 +31,8 @@ def convert_numpy_dtype_to_lamin_feature_type(dtype) -> str:
     orig_type = dtype.name
     # strip precision qualifiers
     type = "".join(i for i in orig_type if not i.isdigit())
-    if type == "int" or type == "float":
-        type = "number"
-    elif type == "object" or type == "str":
-        type = "category"
+    if type == "object" or type == "str":
+        type = "cat"
     return type
 
 
@@ -42,38 +43,44 @@ def __init__(self, *args, **kwargs):
     # now we proceed with the user-facing constructor
     if len(args) != 0:
         raise ValueError("Only non-keyword args allowed")
-    type: Optional[Union[type, str]] = (  # noqa
-        kwargs.pop("type") if "type" in kwargs else None
-    )
-    registries: list[Registry] | None = (
-        kwargs.pop("registries") if "registries" in kwargs else None
-    )
+    dtype: type | str = kwargs.pop("dtype") if "dtype" in kwargs else None
     # cast type
-    type_str = None
-    if type is not None:
-        type_str = type.__name__ if not isinstance(type, str) else type
-    if type_str is None:
-        raise ValueError("Please specify a type!")
-    type_str = FEATURE_TYPES.get(type_str, type_str)
-    if type_str not in {"number", "category", "bool"}:
-        raise ValueError("type has to be one of 'number', 'category', 'bool'!")
-    kwargs["type"] = type_str
-    # cast registries
-    registries_str: str | None = None
-    if registries is not None:
-        if isinstance(registries, str):
-            # TODO: add more validation
-            registries_str = registries
+    if dtype is None:
+        raise ValueError("Please pass a type!")
+    elif dtype is not None:
+        if not isinstance(dtype, str):
+            if not isinstance(dtype, list) and dtype.__name__ in FEATURE_TYPES:
+                dtype_str = FEATURE_TYPES[dtype.__name__]
+            else:
+                if not isinstance(dtype, list):
+                    raise ValueError("dtype has to be a list of Registry types")
+                registries_str = ""
+                for cls in dtype:
+                    if not hasattr(cls, "__get_name_with_schema__"):
+                        raise ValueError(
+                            "each element of the list has to be a Registry"
+                        )
+                    registries_str += cls.__get_name_with_schema__() + "|"
+                dtype_str = f'cat[{registries_str.rstrip("|")}]'
         else:
-            if not isinstance(registries, List):
-                raise ValueError("registries has to be a list of Registry types")
-            registries_str = ""
-            for cls in registries:
-                if not hasattr(cls, "__get_name_with_schema__"):
-                    raise ValueError("each element of the list has to be a Registry")
-                registries_str += cls.__get_name_with_schema__() + "|"
-            registries_str = registries_str.rstrip("|")
-    kwargs["registries"] = registries_str
+            dtype_str = dtype
+            # add validation that a registry actually exists
+            if dtype_str not in FEATURE_TYPES.values() and not dtype_str.startswith(
+                "cat"
+            ):
+                raise ValueError(
+                    f"dtype is {dtype_str} but has to be one of 'number', 'int', 'float', 'cat', 'bool', 'cat[...]'!"
+                )
+            if dtype_str != "cat" and dtype_str.startswith("cat"):
+                registries_str = dtype_str.replace("cat[", "").rstrip("]")
+                if registries_str != "":
+                    registry_str_list = registries_str.split("|")
+                    for registry_str in registry_str_list:
+                        if registry_str not in dict_schema_name_to_model_name(Artifact):
+                            raise ValueError(
+                                f"'{registry_str}' is an invalid dtype, pass, e.g. `[ln.ULabel, bt.CellType]` or similar"
+                            )
+    kwargs["dtype"] = dtype_str
     super(Feature, self).__init__(*args, **kwargs)
 
 
@@ -99,11 +106,11 @@ def from_df(cls, df: pd.DataFrame, field: FieldAttr | None = None) -> RecordsLis
     field = Feature.name if field is None else field
     categoricals = categoricals_from_df(df)
 
-    types = {}
+    dtypes = {}
     # categoricals_with_unmapped_categories = {}  # type: ignore
     for name, col in df.items():
         if name in categoricals:
-            types[name] = "category"
+            dtypes[name] = "cat"
             # below is a harder feature to write, now, because it requires to
             # query the link tables between the label Registry and file or collection
             # the original implementation fell short
@@ -117,7 +124,7 @@ def from_df(cls, df: pd.DataFrame, field: FieldAttr | None = None) -> RecordsLis
             #     feature=name
             # ).inspect(categories, "name", logging=False)["not_mapped"]
         else:
-            types[name] = convert_numpy_dtype_to_lamin_feature_type(col.dtype)
+            dtypes[name] = convert_numpy_dtype_to_lamin_feature_type(col.dtype)
 
     # silence the warning "loaded record with exact same name "
     verbosity = settings.verbosity
@@ -128,7 +135,7 @@ def from_df(cls, df: pd.DataFrame, field: FieldAttr | None = None) -> RecordsLis
         if registry != Feature:
             raise ValueError("field must be a Feature FieldAttr!")
         # create records for all features including non-validated
-        features = [Feature(name=name, type=type) for name, type in types.items()]
+        features = [Feature(name=name, dtype=dtype) for name, dtype in dtypes.items()]
     finally:
         settings.verbosity = verbosity
 
@@ -174,9 +181,10 @@ def from_df(cls, df: pd.DataFrame, field: FieldAttr | None = None) -> RecordsLis
 
 
 @doc_args(Feature.save.__doc__)
-def save(self, *args, **kwargs) -> None:
+def save(self, *args, **kwargs) -> Feature:
     """{}."""
     super(Feature, self).save(*args, **kwargs)
+    return self
 
 
 METHOD_NAMES = [

lamindb/_feature_set.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Iterable
+from typing import TYPE_CHECKING, Iterable, Type
 
 import lamindb_setup as ln_setup
 import numpy as np
@@ -14,6 +14,11 @@ from lamindb._utils import attach_func_to_class_method
 
 from ._feature import convert_numpy_dtype_to_lamin_feature_type
 from ._registry import init_self_from_db
+from .core.exceptions import ValidationError
+from .core.schema import (
+    dict_related_model_to_related_name,
+    get_related_name,
+)
 
 if TYPE_CHECKING:
     import pandas as pd
@@ -21,57 +26,7 @@ if TYPE_CHECKING:
     from ._query_set import QuerySet
 
 NUMBER_TYPE = "number"
-
-
-def dict_related_model_to_related_name(orm):
-    d: dict = {
-        i.related_model.__get_name_with_schema__(): i.related_name
-        for i in orm._meta.related_objects
-        if i.related_name is not None
-    }
-    d.update(
-        {
-            i.related_model.__get_name_with_schema__(): i.name
-            for i in orm._meta.many_to_many
-            if i.name is not None
-        }
-    )
-
-    return d
-
-
-def dict_schema_name_to_model_name(orm):
-    d: dict = {
-        i.related_model.__get_name_with_schema__(): i.related_model
-        for i in orm._meta.related_objects
-        if i.related_name is not None
-    }
-    d.update(
-        {
-            i.related_model.__get_name_with_schema__(): i.related_model
-            for i in orm._meta.many_to_many
-            if i.name is not None
-        }
-    )
-
-    return d
-
-
-def get_related_name(features_type: Registry):
-    candidates = [
-        field.related_name
-        for field in FeatureSet._meta.related_objects
-        if field.related_model == features_type
-    ]
-    if not candidates:
-        raise ValueError(
-            f"Can't create feature sets from {features_type.__name__} because it's not"
-            " related to it!\nYou need to create a link model between FeatureSet and"
-            " your Registry in your custom schema.\nTo do so, add a"
-            " line:\nfeature_sets = models.ManyToMany(FeatureSet,"
-            " related_name='mythings')\n"
-        )
-    return candidates[0]
+DICT_KEYS_TYPE = type({}.keys())  # type: ignore
 
 
 def validate_features(features: list[Registry]) -> Registry:
@@ -106,14 +61,14 @@ def __init__(self, *args, **kwargs):
     if len(args) > 1:
         raise ValueError("Only one non-keyword arg allowed: features")
     features: Iterable[Registry] = kwargs.pop("features") if len(args) == 0 else args[0]
-    type: str | None = kwargs.pop("type") if "type" in kwargs else None
+    dtype: str | None = kwargs.pop("dtype") if "dtype" in kwargs else None
     name: str | None = kwargs.pop("name") if "name" in kwargs else None
     if len(kwargs) > 0:
         raise ValueError("Only features, type, name are valid keyword arguments")
     # now code
     features_registry = validate_features(features)
-    if type is None:
-        type = None if features_registry == Feature else NUMBER_TYPE
+    if dtype is None:
+        dtype = None if features_registry == Feature else NUMBER_TYPE
     n_features = len(features)
     features_hash = hash_set({feature.uid for feature in features})
     feature_set = FeatureSet.filter(hash=features_hash).one_or_none()
@@ -128,7 +83,7 @@ def __init__(self, *args, **kwargs):
     super(FeatureSet, self).__init__(
         uid=ids.base62_20(),
         name=name,
-        type=get_type_str(type),
+        dtype=get_type_str(dtype),
         n=n_features,
         registry=features_registry.__get_name_with_schema__(),
         hash=hash,
@@ -144,13 +99,11 @@ def save(self, *args, **kwargs) -> None:
         getattr(self, related_name).set(records)
 
 
-def get_type_str(type: str | None) -> str | None:
-    if type is not None:
-        type_str = type.__name__ if not isinstance(type, str) else type  # type: ignore
+def get_type_str(dtype: str | None) -> str | None:
+    if dtype is not None:
+        type_str = dtype.__name__ if not isinstance(dtype, str) else dtype  # type: ignore
     else:
         type_str = None
-    if type == "int" or type == "float":
-        type_str = NUMBER_TYPE
     return type_str
 
 
@@ -165,7 +118,8 @@ def from_values(
     mute: bool = False,
     organism: Registry | str | None = None,
     public_source: Registry | None = None,
-) -> FeatureSet | None:
+    raise_validation_error: bool = True,
+) -> FeatureSet:
     """{}."""
     if not isinstance(field, FieldAttr):
         raise TypeError(
@@ -173,16 +127,25 @@ def from_values(
         )
     if len(values) == 0:
         raise ValueError("Provide a list of at least one value")
+    if isinstance(values, DICT_KEYS_TYPE):
+        values = list(values)
     registry = field.field.model
     if registry != Feature and type is None:
         type = NUMBER_TYPE
         logger.debug("setting feature set to 'number'")
     validated = registry.validate(values, field=field, mute=mute, organism=organism)
-    if validated.sum() == 0:
-        if mute is True:
-            logger.warning("no validated features, skip creating feature set")
-        return None
-    validated_values = np.array(values)[validated]
+    values_array = np.array(values)
+    validated_values = values_array[validated]
+    if validated.sum() != len(values):
+        not_validated_values = values_array[~validated]
+        msg = (
+            f"These values could not be validated: {not_validated_values.tolist()}\n"
+            f"If there are no typos, add them to their registry: {registry}"
+        )
+        if raise_validation_error:
+            raise ValidationError(msg)
+        elif len(validated_values) == 0:
+            return None  # temporarily return None here
     validated_features = registry.from_values(
         validated_values,
         field=field,
@@ -192,7 +155,7 @@ def from_values(
     feature_set = FeatureSet(
         features=validated_features,
         name=name,
-        type=get_type_str(type),
+        dtype=get_type_str(type),
     )
     return feature_set
 
@@ -217,12 +180,12 @@ def from_df(
         return None
     if registry == Feature:
         validated_features = Feature.from_df(df.loc[:, validated])
-        feature_set = FeatureSet(validated_features, name=name, type=None)
+        feature_set = FeatureSet(validated_features, name=name, dtype=None)
     else:
         dtypes = [col.dtype for (_, col) in df.loc[:, validated].items()]
         if len(set(dtypes)) != 1:
             raise ValueError(f"data types are heterogeneous: {set(dtypes)}")
-        type = convert_numpy_dtype_to_lamin_feature_type(dtypes[0])
+        dtype = convert_numpy_dtype_to_lamin_feature_type(dtypes[0])
         validated_features = registry.from_values(
             df.columns[validated],
             field=field,
@@ -232,7 +195,7 @@ def from_df(
         feature_set = FeatureSet(
             features=validated_features,
             name=name,
-            type=get_type_str(type),
+            dtype=get_type_str(dtype),
         )
     return feature_set
 
@@ -246,14 +209,14 @@ def members(self) -> QuerySet:
         # need to fix this
         return self._features[1]
     related_name = self._get_related_name()
+    if related_name is None:
+        related_name = "features"
     return self.__getattribute__(related_name).all()
 
 
 def _get_related_name(self: FeatureSet) -> str:
-    key_split = self.registry.split(".")
-    orm_name_with_schema = f"{key_split[0]}.{key_split[1]}"
     feature_sets_related_models = dict_related_model_to_related_name(self)
-    related_name = feature_sets_related_models.get(orm_name_with_schema)
+    related_name = feature_sets_related_models.get(self.registry)
     return related_name