lamindb 0.69.8__py3-none-any.whl → 0.69.10__py3-none-any.whl

lamindb/_artifact.py

@@ -22,7 +22,6 @@ from lamindb_setup.core.upath import (
 from lnschema_core import Artifact, Run, Storage
 from lnschema_core.models import IsTree
 from lnschema_core.types import (
-    DataLike,
     VisibilityChoice,
 )
 
@@ -52,9 +51,11 @@ from .core._data import (
     save_feature_sets,
 )
 from .core.storage.file import AUTO_KEY_PREFIX
+from .core.storage.object import _mudata_is_installed
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
+    from mudata import MuData
 
     from lamindb.core.storage._backed_access import AnnDataAccessor, BackedAccessor
 
@@ -114,7 +115,7 @@ def process_pathlike(
 
 def process_data(
     provisional_uid: str,
-    data: UPathStr | DataLike,
+    data: UPathStr | pd.DataFrame | AnnData,
     format: str | None,
     key: str | None,
     default_storage: Storage,
@@ -123,6 +124,13 @@ def process_data(
 ) -> tuple[Any, Path | UPath, str, Storage, bool]:
     """Serialize a data object that's provided as file or in memory."""
     # if not overwritten, data gets stored in default storage
+    if _mudata_is_installed():
+        from mudata import MuData
+
+        data_types = (pd.DataFrame, AnnData, MuData)
+    else:
+        data_types = (pd.DataFrame, AnnData)  # type:ignore
+
     if isinstance(data, (str, Path, UPath)):  # UPathStr, spelled out
         access_token = (
             default_storage._access_token
@@ -138,7 +146,7 @@ def process_data(
         )
         suffix = extract_suffix_from_path(path)
         memory_rep = None
-    elif isinstance(data, (pd.DataFrame, AnnData)):  # DataLike, spelled out
+    elif isinstance(data, data_types):
         storage = default_storage
         memory_rep = data
         if key is not None:
@@ -297,7 +305,7 @@ def get_relative_path_to_directory(
 
 def get_artifact_kwargs_from_data(
     *,
-    data: Path | UPath | str | pd.DataFrame | AnnData,
+    data: Path | UPath | str | pd.DataFrame | AnnData | MuData,
     key: str | None,
     run: Run | None,
     format: str | None,
@@ -427,22 +435,20 @@ def log_storage_hint(
     logger.hint(hint)
 
 
-def data_is_anndata(data: DataLike):
+def data_is_anndata(data: AnnData | UPathStr):
     if isinstance(data, AnnData):
         return True
     if isinstance(data, (str, Path, UPath)):
         return Path(data).suffix in {".h5ad", ".zrad"}
-    return False  # pragma: no cover
+    return False
 
 
-def data_is_mudata(data: DataLike):  # pragma: no cover
-    try:
+def data_is_mudata(data: MuData | UPathStr):
+    if _mudata_is_installed():
         from mudata import MuData
-    except ModuleNotFoundError:
-        return False
 
-    if isinstance(data, MuData):
-        return True
+        if isinstance(data, MuData):
+            return True
     if isinstance(data, (str, Path, UPath)):
         return Path(data).suffix in {".h5mu"}
     return False
@@ -456,6 +462,9 @@ def _check_accessor_artifact(data: Any, accessor: str | None = None):
         elif data_is_anndata(data):
             logger.warning("data is an AnnData, please use .from_anndata()")
             accessor = "AnnData"
+        elif data_is_mudata(data):
+            logger.warning("data is a MuData, please use .from_mudata()")
+            accessor = "MuData"
         else:
             raise TypeError("data has to be a string, Path, UPath")
     return accessor
@@ -620,6 +629,32 @@ def from_anndata(
     return artifact
 
 
+@classmethod  # type: ignore
+@doc_args(Artifact.from_mudata.__doc__)
+def from_mudata(
+    cls,
+    mdata: MuData,
+    key: str | None = None,
+    description: str | None = None,
+    run: Run | None = None,
+    version: str | None = None,
+    is_new_version_of: Artifact | None = None,
+    **kwargs,
+) -> Artifact:
+    """{}."""
+    artifact = Artifact(
+        data=mdata,
+        key=key,
+        run=run,
+        description=description,
+        version=version,
+        is_new_version_of=is_new_version_of,
+        accessor="MuData",
+        **kwargs,
+    )
+    return artifact
+
+
 @classmethod  # type: ignore
 @doc_args(Artifact.from_dir.__doc__)
 def from_dir(
@@ -725,7 +760,7 @@ def from_dir(
 # docstring handled through attach_func_to_class_method
 def replace(
     self,
-    data: UPathStr | DataLike,
+    data: UPathStr,
     run: Run | None = None,
     format: str | None = None,
 ) -> None:
@@ -808,9 +843,7 @@ def backed(self, is_run_input: bool | None = None) -> AnnDataAccessor | BackedAc
 
 
 # docstring handled through attach_func_to_class_method
-def load(
-    self, is_run_input: bool | None = None, stream: bool = False, **kwargs
-) -> DataLike:
+def load(self, is_run_input: bool | None = None, stream: bool = False, **kwargs) -> Any:
     _track_run_input(self, is_run_input)
     if hasattr(self, "_memory_rep") and self._memory_rep is not None:
         return self._memory_rep
@@ -963,6 +996,7 @@ METHOD_NAMES = [
     "__init__",
     "from_anndata",
     "from_df",
+    "from_mudata",
     "backed",
     "stage",
     "load",

lamindb/_can_validate.py

@@ -29,7 +29,7 @@ def inspect(
     field: str | StrField | None = None,
     *,
     mute: bool = False,
-    **kwargs,
+    organism: str | Registry | None = None,
 ) -> InspectResult:
     """{}."""
     return _inspect(
@@ -37,7 +37,7 @@ def inspect(
         values=values,
         field=field,
         mute=mute,
-        **kwargs,
+        organism=organism,
     )
 
 
@@ -49,10 +49,10 @@ def validate(
     field: str | StrField | None = None,
     *,
     mute: bool = False,
-    **kwargs,
+    organism: str | Registry | None = None,
 ) -> np.ndarray:
     """{}."""
-    return _validate(cls=cls, values=values, field=field, mute=mute, **kwargs)
+    return _validate(cls=cls, values=values, field=field, mute=mute, organism=organism)
 
 
 def _inspect(
@@ -62,7 +62,7 @@ def _inspect(
     *,
     mute: bool = False,
     using_key: str | None = None,
-    **kwargs,
+    organism: str | Registry | None = None,
 ) -> pd.DataFrame | dict[str, list[str]]:
     """{}."""
     from lamin_utils._inspect import inspect
@@ -77,20 +77,17 @@ def _inspect(
 
     # inspect in the DB
     result_db = inspect(
-        df=_filter_query_based_on_organism(
-            queryset=queryset, organism=kwargs.get("organism")
-        ),
+        df=_filter_query_based_on_organism(queryset=queryset, organism=organism),
         identifiers=values,
         field=field,
         mute=mute,
-        **kwargs,
     )
     nonval = set(result_db.non_validated).difference(result_db.synonyms_mapper.keys())
 
     if len(nonval) > 0 and orm.__get_schema_name__() == "bionty":
         try:
-            bionty_result = orm.public(organism=kwargs.get("organism")).inspect(
-                values=nonval, field=field, mute=True, **kwargs
+            bionty_result = orm.public(organism=organism).inspect(
+                values=nonval, field=field, mute=True
             )
             bionty_validated = bionty_result.validated
             bionty_mapper = bionty_result.synonyms_mapper
@@ -146,7 +143,7 @@ def _validate(
     *,
     mute: bool = False,
     using_key: str | None = None,
-    **kwargs,
+    organism: str | Registry | None = None,
 ) -> np.ndarray:
     """{}."""
     from lamin_utils._inspect import validate
@@ -161,7 +158,7 @@ def _validate(
     field_values = pd.Series(
         _filter_query_based_on_organism(
             queryset=queryset,
-            organism=kwargs.get("organism"),
+            organism=organism,
             values_list_field=field,
         ),
         dtype="object",
@@ -173,7 +170,6 @@ def _validate(
         case_sensitive=True,
         mute=mute,
         field=field,
-        **kwargs,
     )
     if return_str and len(result) == 1:
         return result[0]
@@ -195,7 +191,7 @@ def standardize(
     public_aware: bool = True,
     keep: Literal["first", "last", False] = "first",
     synonyms_field: str = "synonyms",
-    **kwargs,
+    organism: str | Registry | None = None,
 ) -> list[str] | dict[str, str]:
     """{}."""
     return _standardize(
@@ -209,7 +205,7 @@ def standardize(
         public_aware=public_aware,
         keep=keep,
         synonyms_field=synonyms_field,
-        **kwargs,
+        organism=organism,
     )
 
 
@@ -258,7 +254,7 @@ def _standardize(
     keep: Literal["first", "last", False] = "first",
     synonyms_field: str = "synonyms",
     using_key: str | None = None,
-    **kwargs,
+    organism: str | Registry | None = None,
 ) -> list[str] | dict[str, str]:
     """{}."""
     from lamin_utils._standardize import standardize as map_synonyms
@@ -274,7 +270,6 @@ def _standardize(
     queryset = _queryset(cls, using_key)
     orm = queryset.model
 
-    organism = kwargs.get("organism")
     if _has_organism_field(orm):
         # here, we can safely import lnschema_bionty
         from lnschema_bionty._bionty import create_or_get_organism_record

lamindb/_collection.py

@@ -16,7 +16,7 @@ from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
 from lamindb_setup.core.hashing import hash_set
 from lnschema_core.models import Collection, CollectionArtifact, FeatureSet
-from lnschema_core.types import DataLike, VisibilityChoice
+from lnschema_core.types import VisibilityChoice
 
 from lamindb._utils import attach_func_to_class_method
 from lamindb.core._data import _track_run_input
@@ -40,17 +40,6 @@ if TYPE_CHECKING:
     from ._query_set import QuerySet
 
 
-def _check_accessor_collection(data: Any, accessor: str | None = None):
-    if accessor is None and isinstance(data, (AnnData, pd.DataFrame)):
-        if isinstance(data, pd.DataFrame):
-            logger.warning("data is a DataFrame, please use .from_df()")
-            accessor = "DataFrame"
-        elif data_is_anndata(data):
-            logger.warning("data is an AnnData, please use .from_anndata()")
-            accessor = "AnnData"
-    return accessor
-
-
 def __init__(
     collection: Collection,
     *args,
@@ -61,11 +50,11 @@ def __init__(
         return None
     # now we proceed with the user-facing constructor
     if len(args) > 1:
-        raise ValueError("Only one non-keyword arg allowed: data")
-    data: Artifact | Iterable[Artifact] = (
-        kwargs.pop("data") if len(args) == 0 else args[0]
+        raise ValueError("Only one non-keyword arg allowed: artifacts")
+    artifacts: Artifact | Iterable[Artifact] = (
+        kwargs.pop("artifacts") if len(args) == 0 else args[0]
     )
-    meta: str | None = kwargs.pop("meta") if "meta" in kwargs else None
+    meta: Artifact | None = kwargs.pop("meta") if "meta" in kwargs else None
     name: str | None = kwargs.pop("name") if "name" in kwargs else None
     description: str | None = (
         kwargs.pop("description") if "description" in kwargs else None
@@ -87,14 +76,10 @@ def __init__(
     feature_sets: dict[str, FeatureSet] = (
         kwargs.pop("feature_sets") if "feature_sets" in kwargs else {}
     )
-    accessor = kwargs.pop("accessor") if "accessor" in kwargs else None
-    if not isinstance(data, (Artifact, Iterable)):
-        accessor = _check_accessor_collection(data=data, accessor=accessor)
     if not len(kwargs) == 0:
         raise ValueError(
-            f"Only data, name, run, description, reference, reference_type, visibility can be passed, you passed: {kwargs}"
+            f"Only artifacts, name, run, description, reference, reference_type, visibility can be passed, you passed: {kwargs}"
         )
-
     if is_new_version_of is None:
         provisional_uid = init_uid(version=version, n_full_id=20)
     else:
@@ -104,13 +89,13 @@ def __init__(
         if name is None:
             name = is_new_version_of.name
     run = get_run(run)
-    if isinstance(data, Artifact):
-        data = [data]
+    if isinstance(artifacts, Artifact):
+        artifacts = [artifacts]
     else:
-        if not hasattr(data, "__getitem__"):
+        if not hasattr(artifacts, "__getitem__"):
             raise ValueError("Artifact or List[Artifact] is allowed.")
-        assert isinstance(data[0], Artifact)  # type: ignore
-    hash, feature_sets = from_artifacts(data)  # type: ignore
+        assert isinstance(artifacts[0], Artifact)  # type: ignore
+    hash, feature_sets = from_artifacts(artifacts)  # type: ignore
     if meta is not None:
         if not isinstance(meta, Artifact):
             raise ValueError("meta has to be an Artifact")
@@ -153,12 +138,12 @@ def __init__(
             visibility=visibility,
             **kwargs,
         )
-    collection._artifacts = data
+    collection._artifacts = artifacts
     collection._feature_sets = feature_sets
     # register provenance
     if is_new_version_of is not None:
         _track_run_input(is_new_version_of, run=run)
-    _track_run_input(data, run=run)
+    _track_run_input(artifacts, run=run)
 
 
 # internal function, not exposed to user
@@ -224,7 +209,9 @@ def from_artifacts(artifacts: Iterable[Artifact]) -> tuple[str, dict[str, str]]:
 # docstring handled through attach_func_to_class_method
 def mapped(
     self,
-    label_keys: str | list[str] | None = None,
+    layers_keys: str | list[str] | None = None,
+    obs_keys: str | list[str] | None = None,
+    obsm_keys: str | list[str] | None = None,
     join: Literal["inner", "outer"] | None = "inner",
     encode_labels: bool | list[str] = True,
     unknown_label: str | dict[str, str] | None = None,
@@ -245,7 +232,9 @@ def mapped(
             path_list.append(artifact.path)
     ds = MappedCollection(
         path_list,
-        label_keys,
+        layers_keys,
+        obs_keys,
+        obsm_keys,
         join,
         encode_labels,
         unknown_label,
@@ -273,7 +262,7 @@ def load(
     join: Literal["inner", "outer"] = "outer",
     is_run_input: bool | None = None,
     **kwargs,
-) -> DataLike:
+) -> Any:
     # cannot call _track_run_input here, see comment further down
     all_artifacts = self.artifacts.all()
     suffixes = [artifact.suffix for artifact in all_artifacts]
@@ -321,7 +310,7 @@ def delete(self, permanent: bool | None = None) -> None:
 
 
 # docstring handled through attach_func_to_class_method
-def save(self, *args, **kwargs) -> None:
+def save(self, transfer_labels: bool = False, using: str | None = None) -> None:
     if self.artifact is not None:
         self.artifact.save()
     # we don't need to save feature sets again
@@ -330,18 +319,21 @@ def save(self, *args, **kwargs) -> None:
     # we don't allow updating the collection of artifacts
     # if users want to update the set of artifacts, they
     # have to create a new collection
-    if hasattr(self, "_artifacts"):
-        if self._artifacts is not None and len(self._artifacts) > 0:
-            links = [
-                CollectionArtifact(collection_id=self.id, artifact_id=artifact.id)
-                for artifact in self._artifacts
-            ]
-            # the below seems to preserve the order of the list in the
-            # auto-incrementing integer primary
-            # merely using .unordered_artifacts.set(*...) doesn't achieve this
-            # we need ignore_conflicts=True so that this won't error if links already exist
-            CollectionArtifact.objects.bulk_create(links, ignore_conflicts=True)
+    links = [
+        CollectionArtifact(collection_id=self.id, artifact_id=artifact.id)
+        for artifact in self._artifacts
+    ]
+    # the below seems to preserve the order of the list in the
+    # auto-incrementing integer primary
+    # merely using .unordered_artifacts.set(*...) doesn't achieve this
+    # we need ignore_conflicts=True so that this won't error if links already exist
+    CollectionArtifact.objects.bulk_create(links, ignore_conflicts=True)
     save_feature_set_links(self)
+    if using is not None:
+        logger.warning("using argument is ignored")
+    if transfer_labels:
+        for artifact in self._artifacts:
+            self.labels.add_from(artifact)
 
 
 # docstring handled through attach_func_to_class_method

lamindb/_feature_set.py

@@ -162,7 +162,9 @@ def from_values(
     field: FieldAttr = Feature.name,
     type: str | None = None,
     name: str | None = None,
-    **kwargs,
+    mute: bool = False,
+    organism: Registry | str | None = None,
+    public_source: Registry | None = None,
 ) -> FeatureSet | None:
     """{}."""
     if not isinstance(field, FieldAttr):
@@ -175,13 +177,18 @@ def from_values(
     if registry != Feature and type is None:
         type = NUMBER_TYPE
         logger.debug("setting feature set to 'number'")
-    validated = registry.validate(values, field=field, organism=kwargs.get("organism"))
+    validated = registry.validate(values, field=field, mute=mute, organism=organism)
     if validated.sum() == 0:
-        if kwargs.get("mute") is True:
+        if mute is True:
             logger.warning("no validated features, skip creating feature set")
         return None
     validated_values = np.array(values)[validated]
-    validated_features = registry.from_values(validated_values, field=field, **kwargs)
+    validated_features = registry.from_values(
+        validated_values,
+        field=field,
+        organism=organism,
+        public_source=public_source,
+    )
     feature_set = FeatureSet(
         features=validated_features,
         name=name,
@@ -197,13 +204,15 @@ def from_df(
     df: pd.DataFrame,
     field: FieldAttr = Feature.name,
     name: str | None = None,
-    **kwargs,
+    mute: bool = False,
+    organism: Registry | str | None = None,
+    public_source: Registry | None = None,
 ) -> FeatureSet | None:
     """{}."""
     registry = field.field.model
-    validated = registry.validate(df.columns, field=field, **kwargs)
+    validated = registry.validate(df.columns, field=field, mute=mute, organism=organism)
     if validated.sum() == 0:
-        if kwargs.get("mute") is True:
+        if mute is True:
             logger.warning("no validated features, skip creating feature set")
         return None
     if registry == Feature:
@@ -215,7 +224,10 @@ def from_df(
             raise ValueError(f"data types are heterogeneous: {set(dtypes)}")
         type = convert_numpy_dtype_to_lamin_feature_type(dtypes[0])
         validated_features = registry.from_values(
-            df.columns[validated], field=field, **kwargs
+            df.columns[validated],
+            field=field,
+            organism=organism,
+            public_source=public_source,
         )
         feature_set = FeatureSet(
             features=validated_features,

lamindb/_from_values.py

@@ -19,19 +19,26 @@ def get_or_create_records(
     field: StrField,
     *,
     from_public: bool = False,
-    **kwargs,
+    organism: Registry | str | None = None,
+    public_source: Registry | None = None,
 ) -> list[Registry]:
     """Get or create records from iterables."""
     upon_create_search_names = settings.upon_create_search_names
-    settings.upon_create_search_names = False
     feature: Feature = None
+    organism = _get_organism_record(field, organism)
+    kwargs: dict = {}
+    if organism is not None:
+        kwargs["organism"] = organism
+    if public_source is not None:
+        kwargs["public_source"] = public_source
+    settings.upon_create_search_names = False
     try:
         Registry = field.field.model
         iterable_idx = index_iterable(iterable)
 
         # returns existing records & non-existing values
         records, nonexist_values, msg = get_existing_records(
-            iterable_idx=iterable_idx, field=field, kwargs=kwargs
+            iterable_idx=iterable_idx, field=field, **kwargs
         )
 
         # new records to be created based on new values
@@ -78,26 +85,14 @@ def get_or_create_records(
 def get_existing_records(
     iterable_idx: pd.Index,
     field: StrField,
-    kwargs: dict = None,
+    **kwargs,
 ):
-    if kwargs is None:
-        kwargs = {}
     model = field.field.model
     condition: dict = {} if len(kwargs) == 0 else kwargs.copy()
     # existing records matching is agnostic to the bionty source
     if "public_source" in condition:
         condition.pop("public_source")
 
-    if _has_organism_field(model):
-        from lnschema_bionty._bionty import create_or_get_organism_record
-
-        organism_record = create_or_get_organism_record(
-            organism=kwargs.get("organism"), orm=model
-        )
-        if organism_record is not None:
-            kwargs.update({"organism": organism_record})
-            condition.update({"organism": organism_record})
-
     # standardize based on the DB reference
     # log synonyms mapped terms
     result = model.inspect(
@@ -322,3 +317,13 @@ def _has_organism_field(orm: Registry) -> bool:
         return True
     except FieldDoesNotExist:
         return False
+
+
+def _get_organism_record(field: StrField, organism: str | Registry) -> Registry:
+    model = field.field.model
+    if _has_organism_field(model):
+        from lnschema_bionty._bionty import create_or_get_organism_record
+
+        organism_record = create_or_get_organism_record(organism=organism, orm=model)
+        if organism_record is not None:
+            return organism_record

lamindb/_registry.py

@@ -129,7 +129,11 @@ def __init__(orm: Registry, *args, **kwargs):
 @classmethod  # type:ignore
 @doc_args(Registry.from_values.__doc__)
 def from_values(
-    cls, values: ListLike, field: StrField | None = None, **kwargs
+    cls,
+    values: ListLike,
+    field: StrField | None = None,
+    organism: Registry | str | None = None,
+    public_source: Registry | None = None,
 ) -> list[Registry]:
     """{}."""
     from_public = True if cls.__module__.startswith("lnschema_bionty.") else False
@@ -138,7 +142,8 @@ def from_values(
         iterable=values,
         field=getattr(cls, field_str),
         from_public=from_public,
-        **kwargs,
+        organism=organism,
+        public_source=public_source,
     )
 
 

lamindb/core/__init__.py

@@ -14,14 +14,21 @@ Registries:
    LabelManager
    IsTree
    IsVersioned
-   DataFrameAnnotator
-   AnnDataAnnotator
-   AnnotateLookup
    CanValidate
    HasParents
    InspectResult
    fields
 
+Annotators:
+
+.. autosummary::
+   :toctree: .
+
+   DataFrameAnnotator
+   AnnDataAnnotator
+   MuDataAnnotator
+   AnnotateLookup
+
 Classes:
 
 .. autosummary::
@@ -53,7 +60,12 @@ from lnschema_core.models import (
     Registry,
 )
 
-from lamindb._annotate import AnnDataAnnotator, AnnotateLookup, DataFrameAnnotator
+from lamindb._annotate import (
+    AnnDataAnnotator,
+    AnnotateLookup,
+    DataFrameAnnotator,
+    MuDataAnnotator,
+)
 from lamindb._query_manager import QueryManager
 from lamindb._query_set import QuerySet, RecordsList
 from lamindb.core._feature_manager import FeatureManager

lamindb/core/_data.py

@@ -109,17 +109,7 @@ def describe(self: Data):
         else:
             direct_fields.append(f.name)
 
-    # Display Provenance
-    # display line by line the foreign key fields
-    from lamindb._parents import _transform_emoji
-
-    emojis = {
-        "storage": "🗃️",
-        "created_by": "👤",
-        "transform": _transform_emoji(self.transform),
-        "run": "👣",
-        "artifact": "📄",
-    }
+    # provenance
     if len(foreign_key_fields) > 0:  # always True for Artifact and Collection
         record_msg = f"{colors.green(model_name)}{__repr__(self, include_foreign_keys=False).lstrip(model_name)}"
         msg += f"{record_msg}\n\n"
@@ -127,17 +117,16 @@ def describe(self: Data):
         msg += f"{colors.green('Provenance')}:\n  "
         related_msg = "".join(
             [
-                f"{emojis.get(i, '📎')} {i}: {self.__getattribute__(i)}\n  "
-                for i in foreign_key_fields
-                if self.__getattribute__(i) is not None
+                f"📎 {field}: {self.__getattribute__(field)}\n  "
+                for field in foreign_key_fields
+                if self.__getattribute__(field) is not None
             ]
         )
         msg += related_msg
     # input of
-    # can only access many-to-many once record is saved
     if self.id is not None and self.input_of.exists():
         values = [format_field_value(i.started_at) for i in self.input_of.all()]
-        msg += f"⬇️ input_of ({colors.italic('core.Run')}): {values}\n    "
+        msg += f"📎 input_of ({colors.italic('core.Run')}): {values}\n    "
     msg = msg.rstrip(" ")  # do not use removesuffix as we need to remove 2 or 4 spaces
     msg += print_features(self)
     msg += print_labels(self)