lamindb 0.69.9__py3-none-any.whl → 0.70.0__py3-none-any.whl

lamindb/_artifact.py

@@ -51,9 +51,11 @@ from .core._data import (
     save_feature_sets,
 )
 from .core.storage.file import AUTO_KEY_PREFIX
+from .core.storage.object import _mudata_is_installed
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
+    from mudata import MuData
 
     from lamindb.core.storage._backed_access import AnnDataAccessor, BackedAccessor
 
@@ -122,6 +124,13 @@ def process_data(
 ) -> tuple[Any, Path | UPath, str, Storage, bool]:
     """Serialize a data object that's provided as file or in memory."""
     # if not overwritten, data gets stored in default storage
+    if _mudata_is_installed():
+        from mudata import MuData
+
+        data_types = (pd.DataFrame, AnnData, MuData)
+    else:
+        data_types = (pd.DataFrame, AnnData)  # type:ignore
+
     if isinstance(data, (str, Path, UPath)):  # UPathStr, spelled out
         access_token = (
             default_storage._access_token
@@ -137,7 +146,7 @@ def process_data(
         )
         suffix = extract_suffix_from_path(path)
         memory_rep = None
-    elif isinstance(data, (pd.DataFrame, AnnData)):
+    elif isinstance(data, data_types):
         storage = default_storage
         memory_rep = data
         if key is not None:
@@ -229,7 +238,7 @@ def get_stat_or_artifact(
                 "💡 you can make this error a warning:\n"
                 "    ln.settings.upon_artifact_create_if_hash_exists"
             )
-            raise RuntimeError(f"{msg}\n{hint}")
+            raise FileExistsError(f"{msg}\n{hint}")
         elif settings.upon_artifact_create_if_hash_exists == "warn_create_new":
             logger.warning(
                 "creating new Artifact object despite existing artifact with same hash:"
@@ -237,10 +246,12 @@ def get_stat_or_artifact(
             )
             return size, hash, hash_type, n_objects
         else:
-            from_trash = "(from trash)" if result[0].visibility == -1 else ""
-            logger.warning(
-                f"returning existing artifact with same hash{from_trash}: {result[0]}"
-            )
+            if result[0].visibility == -1:
+                raise FileExistsError(
+                    f"You're trying to re-create this artifact in trash: {result[0]}"
+                    "Either permanently delete it with `artifact.delete(permanent=True)` or restore it with `artifact.restore()`"
+                )
+            logger.warning(f"returning existing artifact with same hash: {result[0]}")
             return result[0]
     else:
         return size, hash, hash_type, n_objects
@@ -296,7 +307,7 @@ def get_relative_path_to_directory(
 
 def get_artifact_kwargs_from_data(
     *,
-    data: Path | UPath | str | pd.DataFrame | AnnData,
+    data: Path | UPath | str | pd.DataFrame | AnnData | MuData,
     key: str | None,
     run: Run | None,
     format: str | None,
@@ -322,6 +333,15 @@ def get_artifact_kwargs_from_data(
         using_key=using_key,
     )
     if isinstance(stat_or_artifact, Artifact):
+        # update the run of the existing artifact
+        if run is not None:
+            # save the information that this artifact was previously
+            # produced by another run
+            if stat_or_artifact.run is not None:
+                stat_or_artifact.run.replicated_output_artifacts.add(stat_or_artifact)
+            # update the run of the artifact with the latest run
+            stat_or_artifact.run = run
+            stat_or_artifact.transform = run.transform
         return stat_or_artifact, None
     else:
         size, hash, hash_type, n_objects = stat_or_artifact
@@ -431,17 +451,15 @@ def data_is_anndata(data: AnnData | UPathStr):
         return True
     if isinstance(data, (str, Path, UPath)):
         return Path(data).suffix in {".h5ad", ".zrad"}
-    return False  # pragma: no cover
+    return False
 
 
-def data_is_mudata(data: Any | UPathStr):  # pragma: no cover
-    try:
+def data_is_mudata(data: MuData | UPathStr):
+    if _mudata_is_installed():
         from mudata import MuData
-    except ModuleNotFoundError:
-        return False
 
-    if isinstance(data, MuData):
-        return True
+        if isinstance(data, MuData):
+            return True
     if isinstance(data, (str, Path, UPath)):
         return Path(data).suffix in {".h5mu"}
     return False
@@ -455,6 +473,9 @@ def _check_accessor_artifact(data: Any, accessor: str | None = None):
         elif data_is_anndata(data):
             logger.warning("data is an AnnData, please use .from_anndata()")
             accessor = "AnnData"
+        elif data_is_mudata(data):
+            logger.warning("data is a MuData, please use .from_mudata()")
+            accessor = "MuData"
         else:
             raise TypeError("data has to be a string, Path, UPath")
     return accessor
@@ -619,6 +640,32 @@ def from_anndata(
     return artifact
 
 
+@classmethod  # type: ignore
+@doc_args(Artifact.from_mudata.__doc__)
+def from_mudata(
+    cls,
+    mdata: MuData,
+    key: str | None = None,
+    description: str | None = None,
+    run: Run | None = None,
+    version: str | None = None,
+    is_new_version_of: Artifact | None = None,
+    **kwargs,
+) -> Artifact:
+    """{}."""
+    artifact = Artifact(
+        data=mdata,
+        key=key,
+        run=run,
+        description=description,
+        version=version,
+        is_new_version_of=is_new_version_of,
+        accessor="MuData",
+        **kwargs,
+    )
+    return artifact
+
+
 @classmethod  # type: ignore
 @doc_args(Artifact.from_dir.__doc__)
 def from_dir(
@@ -818,7 +865,7 @@ def load(self, is_run_input: bool | None = None, stream: bool = False, **kwargs)
 
 
 # docstring handled through attach_func_to_class_method
-def stage(self, is_run_input: bool | None = None) -> Path:
+def cache(self, is_run_input: bool | None = None) -> Path:
     _track_run_input(self, is_run_input)
 
     using_key = settings._using_key
@@ -896,10 +943,10 @@ def _delete_skip_storage(artifact, *args, **kwargs) -> None:
 
 
 # docstring handled through attach_func_to_class_method
-def save(self, *args, **kwargs) -> None:
+def save(self, upload: bool | None = None, **kwargs) -> None:
     access_token = kwargs.pop("access_token", None)
 
-    self._save_skip_storage(*args, **kwargs)
+    self._save_skip_storage(**kwargs)
 
     from lamindb._save import check_and_attempt_clearing, check_and_attempt_upload
 
@@ -915,9 +962,9 @@ def save(self, *args, **kwargs) -> None:
         raise RuntimeError(exception)
 
 
-def _save_skip_storage(file, *args, **kwargs) -> None:
+def _save_skip_storage(file, **kwargs) -> None:
     save_feature_sets(file)
-    super(Artifact, file).save(*args, **kwargs)
+    super(Artifact, file).save(**kwargs)
     save_feature_set_links(file)
 
 
@@ -960,8 +1007,9 @@ METHOD_NAMES = [
     "__init__",
     "from_anndata",
     "from_df",
+    "from_mudata",
     "backed",
-    "stage",
+    "cache",
     "load",
     "delete",
     "save",
@@ -987,5 +1035,6 @@ for name in METHOD_NAMES:
 Artifact._delete_skip_storage = _delete_skip_storage
 Artifact._save_skip_storage = _save_skip_storage
 Artifact.path = path
+Artifact.stage = cache
 # this seems a Django-generated function
 delattr(Artifact, "get_visibility_display")

lamindb/_can_validate.py

@@ -29,7 +29,7 @@ def inspect(
     field: str | StrField | None = None,
     *,
     mute: bool = False,
-    **kwargs,
+    organism: str | Registry | None = None,
 ) -> InspectResult:
     """{}."""
     return _inspect(
@@ -37,7 +37,7 @@ def inspect(
         values=values,
         field=field,
         mute=mute,
-        **kwargs,
+        organism=organism,
     )
 
 
@@ -49,10 +49,10 @@ def validate(
     field: str | StrField | None = None,
     *,
     mute: bool = False,
-    **kwargs,
+    organism: str | Registry | None = None,
 ) -> np.ndarray:
     """{}."""
-    return _validate(cls=cls, values=values, field=field, mute=mute, **kwargs)
+    return _validate(cls=cls, values=values, field=field, mute=mute, organism=organism)
 
 
 def _inspect(
@@ -62,7 +62,7 @@ def _inspect(
     *,
     mute: bool = False,
     using_key: str | None = None,
-    **kwargs,
+    organism: str | Registry | None = None,
 ) -> pd.DataFrame | dict[str, list[str]]:
     """{}."""
     from lamin_utils._inspect import inspect
@@ -77,20 +77,17 @@ def _inspect(
 
     # inspect in the DB
     result_db = inspect(
-        df=_filter_query_based_on_organism(
-            queryset=queryset, organism=kwargs.get("organism")
-        ),
+        df=_filter_query_based_on_organism(queryset=queryset, organism=organism),
         identifiers=values,
         field=field,
         mute=mute,
-        **kwargs,
     )
     nonval = set(result_db.non_validated).difference(result_db.synonyms_mapper.keys())
 
     if len(nonval) > 0 and orm.__get_schema_name__() == "bionty":
         try:
-            bionty_result = orm.public(organism=kwargs.get("organism")).inspect(
-                values=nonval, field=field, mute=True, **kwargs
+            bionty_result = orm.public(organism=organism).inspect(
+                values=nonval, field=field, mute=True
             )
             bionty_validated = bionty_result.validated
             bionty_mapper = bionty_result.synonyms_mapper
@@ -146,7 +143,7 @@ def _validate(
     *,
     mute: bool = False,
     using_key: str | None = None,
-    **kwargs,
+    organism: str | Registry | None = None,
 ) -> np.ndarray:
     """{}."""
     from lamin_utils._inspect import validate
@@ -161,7 +158,7 @@ def _validate(
     field_values = pd.Series(
         _filter_query_based_on_organism(
             queryset=queryset,
-            organism=kwargs.get("organism"),
+            organism=organism,
             values_list_field=field,
         ),
         dtype="object",
@@ -173,7 +170,6 @@ def _validate(
         case_sensitive=True,
         mute=mute,
         field=field,
-        **kwargs,
     )
     if return_str and len(result) == 1:
         return result[0]
@@ -195,7 +191,7 @@ def standardize(
     public_aware: bool = True,
     keep: Literal["first", "last", False] = "first",
     synonyms_field: str = "synonyms",
-    **kwargs,
+    organism: str | Registry | None = None,
 ) -> list[str] | dict[str, str]:
     """{}."""
     return _standardize(
@@ -209,7 +205,7 @@ def standardize(
         public_aware=public_aware,
         keep=keep,
         synonyms_field=synonyms_field,
-        **kwargs,
+        organism=organism,
     )
 
 
@@ -258,7 +254,7 @@ def _standardize(
     keep: Literal["first", "last", False] = "first",
     synonyms_field: str = "synonyms",
     using_key: str | None = None,
-    **kwargs,
+    organism: str | Registry | None = None,
 ) -> list[str] | dict[str, str]:
     """{}."""
     from lamin_utils._standardize import standardize as map_synonyms
@@ -274,7 +270,6 @@ def _standardize(
     queryset = _queryset(cls, using_key)
     orm = queryset.model
 
-    organism = kwargs.get("organism")
     if _has_organism_field(orm):
         # here, we can safely import lnschema_bionty
         from lnschema_bionty._bionty import create_or_get_organism_record

lamindb/_collection.py

@@ -40,17 +40,6 @@ if TYPE_CHECKING:
     from ._query_set import QuerySet
 
 
-def _check_accessor_collection(data: Any, accessor: str | None = None):
-    if accessor is None and isinstance(data, (AnnData, pd.DataFrame)):
-        if isinstance(data, pd.DataFrame):
-            logger.warning("data is a DataFrame, please use .from_df()")
-            accessor = "DataFrame"
-        elif data_is_anndata(data):
-            logger.warning("data is an AnnData, please use .from_anndata()")
-            accessor = "AnnData"
-    return accessor
-
-
 def __init__(
     collection: Collection,
     *args,
@@ -61,9 +50,9 @@ def __init__(
         return None
     # now we proceed with the user-facing constructor
     if len(args) > 1:
-        raise ValueError("Only one non-keyword arg allowed: data")
-    data: Artifact | Iterable[Artifact] = (
-        kwargs.pop("data") if len(args) == 0 else args[0]
+        raise ValueError("Only one non-keyword arg allowed: artifacts")
+    artifacts: Artifact | Iterable[Artifact] = (
+        kwargs.pop("artifacts") if len(args) == 0 else args[0]
     )
     meta: Artifact | None = kwargs.pop("meta") if "meta" in kwargs else None
     name: str | None = kwargs.pop("name") if "name" in kwargs else None
@@ -87,14 +76,10 @@ def __init__(
     feature_sets: dict[str, FeatureSet] = (
         kwargs.pop("feature_sets") if "feature_sets" in kwargs else {}
     )
-    accessor = kwargs.pop("accessor") if "accessor" in kwargs else None
-    if not isinstance(data, (Artifact, Iterable)):
-        accessor = _check_accessor_collection(data=data, accessor=accessor)
     if not len(kwargs) == 0:
         raise ValueError(
-            f"Only data, name, run, description, reference, reference_type, visibility can be passed, you passed: {kwargs}"
+            f"Only artifacts, name, run, description, reference, reference_type, visibility can be passed, you passed: {kwargs}"
         )
-
     if is_new_version_of is None:
         provisional_uid = init_uid(version=version, n_full_id=20)
     else:
@@ -104,13 +89,13 @@ def __init__(
         if name is None:
             name = is_new_version_of.name
     run = get_run(run)
-    if isinstance(data, Artifact):
-        data = [data]
+    if isinstance(artifacts, Artifact):
+        artifacts = [artifacts]
     else:
-        if not hasattr(data, "__getitem__"):
+        if not hasattr(artifacts, "__getitem__"):
             raise ValueError("Artifact or List[Artifact] is allowed.")
-        assert isinstance(data[0], Artifact)  # type: ignore
-    hash, feature_sets = from_artifacts(data)  # type: ignore
+        assert isinstance(artifacts[0], Artifact)  # type: ignore
+    hash, feature_sets = from_artifacts(artifacts)  # type: ignore
     if meta is not None:
         if not isinstance(meta, Artifact):
             raise ValueError("meta has to be an Artifact")
@@ -131,6 +116,17 @@ def __init__(
         logger.warning(
             f"returning existing collection with same hash: {existing_collection}"
         )
+        # update the run of the existing artifact
+        if run is not None:
+            # save the information that this artifact was previously
+            # produced by another run
+            if existing_collection.run is not None:
+                existing_collection.run.replicated_output_collections.add(
+                    existing_collection
+                )
+            # update the run of the artifact with the latest run
+            existing_collection.run = run
+            existing_collection.transform = run.transform
         init_self_from_db(collection, existing_collection)
         for slot, feature_set in collection.features._feature_set_by_slot.items():
             if slot in feature_sets:
@@ -153,12 +149,12 @@ def __init__(
             visibility=visibility,
             **kwargs,
         )
-    collection._artifacts = data
+    collection._artifacts = artifacts
     collection._feature_sets = feature_sets
     # register provenance
     if is_new_version_of is not None:
         _track_run_input(is_new_version_of, run=run)
-    _track_run_input(data, run=run)
+    _track_run_input(artifacts, run=run)
 
 
 # internal function, not exposed to user
@@ -224,7 +220,9 @@ def from_artifacts(artifacts: Iterable[Artifact]) -> tuple[str, dict[str, str]]:
 # docstring handled through attach_func_to_class_method
 def mapped(
     self,
-    label_keys: str | list[str] | None = None,
+    layers_keys: str | list[str] | None = None,
+    obs_keys: str | list[str] | None = None,
+    obsm_keys: str | list[str] | None = None,
     join: Literal["inner", "outer"] | None = "inner",
     encode_labels: bool | list[str] = True,
     unknown_label: str | dict[str, str] | None = None,
@@ -240,12 +238,14 @@ def mapped(
             logger.warning(f"Ignoring artifact with suffix {artifact.suffix}")
             continue
         elif not stream:
-            path_list.append(artifact.stage())
+            path_list.append(artifact.cache())
         else:
             path_list.append(artifact.path)
     ds = MappedCollection(
         path_list,
-        label_keys,
+        layers_keys,
+        obs_keys,
+        obsm_keys,
         join,
         encode_labels,
         unknown_label,
@@ -259,11 +259,11 @@ def mapped(
 
 
 # docstring handled through attach_func_to_class_method
-def stage(self, is_run_input: bool | None = None) -> list[UPath]:
+def cache(self, is_run_input: bool | None = None) -> list[UPath]:
     _track_run_input(self, is_run_input)
     path_list = []
     for artifact in self.artifacts.all():
-        path_list.append(artifact.stage())
+        path_list.append(artifact.cache())
     return path_list
 
 
@@ -321,7 +321,7 @@ def delete(self, permanent: bool | None = None) -> None:
 
 
 # docstring handled through attach_func_to_class_method
-def save(self, *args, **kwargs) -> None:
+def save(self, transfer_labels: bool = False, using: str | None = None) -> None:
     if self.artifact is not None:
         self.artifact.save()
     # we don't need to save feature sets again
@@ -330,18 +330,21 @@ def save(self, *args, **kwargs) -> None:
     # we don't allow updating the collection of artifacts
     # if users want to update the set of artifacts, they
     # have to create a new collection
-    if hasattr(self, "_artifacts"):
-        if self._artifacts is not None and len(self._artifacts) > 0:
-            links = [
-                CollectionArtifact(collection_id=self.id, artifact_id=artifact.id)
-                for artifact in self._artifacts
-            ]
-            # the below seems to preserve the order of the list in the
-            # auto-incrementing integer primary
-            # merely using .unordered_artifacts.set(*...) doesn't achieve this
-            # we need ignore_conflicts=True so that this won't error if links already exist
-            CollectionArtifact.objects.bulk_create(links, ignore_conflicts=True)
+    links = [
+        CollectionArtifact(collection_id=self.id, artifact_id=artifact.id)
+        for artifact in self._artifacts
+    ]
+    # the below seems to preserve the order of the list in the
+    # auto-incrementing integer primary
+    # merely using .unordered_artifacts.set(*...) doesn't achieve this
+    # we need ignore_conflicts=True so that this won't error if links already exist
+    CollectionArtifact.objects.bulk_create(links, ignore_conflicts=True)
     save_feature_set_links(self)
+    if using is not None:
+        logger.warning("using argument is ignored")
+    if transfer_labels:
+        for artifact in self._artifacts:
+            self.labels.add_from(artifact)
 
 
 # docstring handled through attach_func_to_class_method
@@ -363,7 +366,7 @@ def artifacts(self) -> QuerySet:
 METHOD_NAMES = [
     "__init__",
     "mapped",
-    "stage",
+    "cache",
     "load",
     "delete",
     "save",
@@ -385,3 +388,4 @@ for name in METHOD_NAMES:
 # this seems a Django-generated function
 delattr(Collection, "get_visibility_display")
 Collection.artifacts = artifacts
+Collection.stage = cache

lamindb/_feature_set.py

@@ -162,7 +162,9 @@ def from_values(
     field: FieldAttr = Feature.name,
     type: str | None = None,
     name: str | None = None,
-    **kwargs,
+    mute: bool = False,
+    organism: Registry | str | None = None,
+    public_source: Registry | None = None,
 ) -> FeatureSet | None:
     """{}."""
     if not isinstance(field, FieldAttr):
@@ -175,13 +177,18 @@ def from_values(
     if registry != Feature and type is None:
         type = NUMBER_TYPE
         logger.debug("setting feature set to 'number'")
-    validated = registry.validate(values, field=field, organism=kwargs.get("organism"))
+    validated = registry.validate(values, field=field, mute=mute, organism=organism)
     if validated.sum() == 0:
-        if kwargs.get("mute") is True:
+        if mute is True:
             logger.warning("no validated features, skip creating feature set")
         return None
     validated_values = np.array(values)[validated]
-    validated_features = registry.from_values(validated_values, field=field, **kwargs)
+    validated_features = registry.from_values(
+        validated_values,
+        field=field,
+        organism=organism,
+        public_source=public_source,
+    )
     feature_set = FeatureSet(
         features=validated_features,
         name=name,
@@ -197,13 +204,15 @@ def from_df(
     df: pd.DataFrame,
     field: FieldAttr = Feature.name,
     name: str | None = None,
-    **kwargs,
+    mute: bool = False,
+    organism: Registry | str | None = None,
+    public_source: Registry | None = None,
 ) -> FeatureSet | None:
     """{}."""
     registry = field.field.model
-    validated = registry.validate(df.columns, field=field, **kwargs)
+    validated = registry.validate(df.columns, field=field, mute=mute, organism=organism)
     if validated.sum() == 0:
-        if kwargs.get("mute") is True:
+        if mute is True:
             logger.warning("no validated features, skip creating feature set")
         return None
     if registry == Feature:
@@ -215,7 +224,10 @@ def from_df(
             raise ValueError(f"data types are heterogeneous: {set(dtypes)}")
         type = convert_numpy_dtype_to_lamin_feature_type(dtypes[0])
         validated_features = registry.from_values(
-            df.columns[validated], field=field, **kwargs
+            df.columns[validated],
+            field=field,
+            organism=organism,
+            public_source=public_source,
         )
         feature_set = FeatureSet(
             features=validated_features,