lamindb 0.70.3__py3-none-any.whl → 0.70.4__py3-none-any.whl

lamindb/__init__.py

@@ -40,7 +40,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.70.3"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.70.4"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 

lamindb/_annotate.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Iterable, Optional
+from typing import TYPE_CHECKING, Iterable
 
 import anndata as ad
 import lamindb_setup as ln_setup
@@ -10,6 +10,7 @@ from lamindb_setup.core._docs import doc_args
 from lnschema_core import Artifact, Collection, Feature, Registry, Run, ULabel
 
 if TYPE_CHECKING:
+    from lamindb_setup.core.types import UPathStr
     from lnschema_core.types import FieldAttr
     from mudata import MuData
 
@@ -244,7 +245,7 @@ class DataFrameAnnotator:
         )
         return self._validated
 
-    def save_artifact(self, description: str, **kwargs) -> Artifact:
+    def save_artifact(self, description: str | None = None, **kwargs) -> Artifact:
         """Save the validated DataFrame and metadata.
 
         Args:
@@ -327,10 +328,10 @@ class DataFrameAnnotator:
 
 
 class AnnDataAnnotator(DataFrameAnnotator):
-    """Annotation flow for an ``AnnData`` object.
+    """Annotation flow for ``AnnData``.
 
     Args:
-        adata: The AnnData object to annotate.
+        data: The AnnData object or an AnnData-like path.
         var_index: The registry field for mapping the ``.var`` index.
         categoricals: A dictionary mapping ``.obs.columns`` to a registry field.
         using: A reference LaminDB instance.
@@ -349,14 +350,29 @@ class AnnDataAnnotator(DataFrameAnnotator):
 
     def __init__(
         self,
-        adata: ad.AnnData,
+        data: ad.AnnData | UPathStr,
         var_index: FieldAttr,
         categoricals: dict[str, FieldAttr] | None = None,
         using: str = "default",
         verbosity: str = "hint",
         organism: str | None = None,
     ) -> None:
-        self._adata = adata
+        from lamindb_setup.core import upath
+
+        from ._artifact import data_is_anndata
+
+        if not data_is_anndata(data):
+            raise ValueError(
+                "data has to be an AnnData object or a path to AnnData-like"
+            )
+        if isinstance(data, ad.AnnData):
+            self._adata = data
+        else:
+            from lamindb.core.storage._backed_access import backed_access
+
+            self._adata = backed_access(upath.create_path(data))
+
+        self._data = data
         self._var_field = var_index
         super().__init__(
             df=self._adata.obs,
@@ -443,7 +459,7 @@ class AnnDataAnnotator(DataFrameAnnotator):
         self._validated = validated_var and validated_obs
         return self._validated
 
-    def save_artifact(self, description: str, **kwargs) -> Artifact:
+    def save_artifact(self, description: str | None = None, **kwargs) -> Artifact:
         """Save the validated ``AnnData`` and metadata.
 
         Args:
@@ -457,7 +473,8 @@ class AnnDataAnnotator(DataFrameAnnotator):
             raise ValidationError("Please run `validate()` first!")
 
         self._artifact = save_artifact(
-            self._adata,
+            self._data,
+            adata=self._adata,
             description=description,
             columns_field=self.var_index,
             fields=self.categoricals,
@@ -697,7 +714,7 @@ class MuDataAnnotator:
         self._validated = validated_var and validated_obs
         return self._validated
 
-    def save_artifact(self, description: str, **kwargs) -> Artifact:
+    def save_artifact(self, description: str | None = None, **kwargs) -> Artifact:
         """Save the validated ``MuData`` and metadata.
 
         Args:
@@ -749,7 +766,7 @@ class Annotate:
     @doc_args(AnnDataAnnotator.__doc__)
     def from_anndata(
         cls,
-        adata: ad.AnnData,
+        data: ad.AnnData | UPathStr,
         var_index: FieldAttr,
         categoricals: dict[str, FieldAttr] | None = None,
         using: str = "default",
@@ -758,7 +775,7 @@ class Annotate:
     ) -> AnnDataAnnotator:
         """{}."""
         return AnnDataAnnotator(
-            adata=adata,
+            data=data,
             var_index=var_index,
             categoricals=categoricals,
             using=using,
@@ -920,10 +937,11 @@ def validate_categories_in_df(
 
 def save_artifact(
     data: pd.DataFrame | ad.AnnData | MuData,
-    description: str,
     fields: dict[str, FieldAttr] | dict[str, dict[str, FieldAttr]],
     columns_field: FieldAttr | dict[str, FieldAttr],
+    description: str | None = None,
     organism: str | None = None,
+    adata: ad.AnnData | None = None,
     **kwargs,
 ) -> Artifact:
     """Save all metadata with an Artifact.
@@ -934,15 +952,21 @@ def save_artifact(
         fields: A dictionary mapping obs_column to registry_field.
         columns_field: The registry field to validate variables index against.
         organism: The organism name.
+        adata: The AnnData object to save, must be provided if data is a path.
         kwargs: Additional keyword arguments to pass to the registry model.
 
     Returns:
         The saved Artifact.
     """
+    from ._artifact import data_is_anndata
+
     artifact = None
-    if isinstance(data, ad.AnnData):
+    if data_is_anndata(data):
+        assert adata is not None
         artifact = Artifact.from_anndata(data, description=description, **kwargs)
-        artifact.n_observations = data.n_obs
+        artifact.n_observations = adata.shape[0]
+        data = adata
+
     elif isinstance(data, pd.DataFrame):
         artifact = Artifact.from_df(data, description=description, **kwargs)
     else:

lamindb/_artifact.py

@@ -463,7 +463,7 @@ def data_is_anndata(data: AnnData | UPathStr):
             if ".anndata" in data_path.suffixes:
                 return True
             # check only for local, expensive for cloud
-            if fsspec.utils.get_protocol(data_path) == "file":
+            if fsspec.utils.get_protocol(data_path.as_posix()) == "file":
                 return zarr_is_adata(data_path)
             else:
                 logger.warning("We do not check if cloud zarr is AnnData or not.")
@@ -597,6 +597,18 @@ def __init__(artifact: Artifact, *args, **kwargs):
     else:
         kwargs = kwargs_or_artifact
 
+    # in case we have a new version of a folder with a different hash, print a
+    # warning that the old version can't be recovered
+    if (
+        is_new_version_of is not None
+        and is_new_version_of.n_objects is not None
+        and is_new_version_of.n_objects > 1
+    ):
+        logger.warning(
+            f"artifact version {version} will _update_ the state of folder {is_new_version_of.path} - "
+            "to _retain_ the old state by duplicating the entire folder, do _not_ pass `is_new_version_of`"
+        )
+
     kwargs["uid"] = provisional_uid
     kwargs["version"] = version
     kwargs["description"] = description
@@ -652,7 +664,7 @@ def from_df(
 @doc_args(Artifact.from_anndata.__doc__)
 def from_anndata(
     cls,
-    adata: AnnData,
+    adata: AnnData | UPathStr,
     key: str | None = None,
     description: str | None = None,
     run: Run | None = None,
@@ -661,6 +673,8 @@ def from_anndata(
     **kwargs,
 ) -> Artifact:
     """{}."""
+    if not data_is_anndata(adata):
+        raise ValueError("data has to be an AnnData object or a path to AnnData-like")
     artifact = Artifact(
         data=adata,
         key=key,
@@ -916,7 +930,7 @@ def delete(
 ) -> None:
     # by default, we only move artifacts into the trash (visibility = -1)
     trash_visibility = VisibilityChoice.trash.value
-    if self.visibility > trash_visibility and permanent is not True:
+    if self.visibility > trash_visibility and not permanent:
         if storage is not None:
             logger.warning("moving artifact to trash, storage arg is ignored")
         # move to trash
@@ -935,41 +949,44 @@ def delete(
         )
         delete_record = response == "y"
     else:
-        # this second option doesn't feel very intuitive
-        delete_record = permanent
+        assert permanent
+        delete_record = True
 
-    if delete_record is True:
+    if delete_record:
         # need to grab file path before deletion
         try:
-            filepath = filepath_from_artifact(self, using_key)
+            path = filepath_from_artifact(self, using_key)
         except OSError:
             # we can still delete the record
+            logger.warning("Could not get path")
             storage = False
         # only delete in storage if DB delete is successful
         # DB delete might error because of a foreign key constraint violated etc.
         self._delete_skip_storage()
         if self.key is None or self.key_is_virtual:
-            # always delete in storage if the key is virtual
-            delete_in_storage = True
-            if storage is not None:
-                logger.warning("storage arg is ignored if storage key is non-semantic")
+            # do not ask for confirmation also if storage is None
+            delete_in_storage = storage is None or storage
         else:
             # for artifacts with non-virtual semantic storage keys (key is not None)
             # ask for extra-confirmation
             if storage is None:
                 response = input(
-                    f"Are you sure to want to delete {filepath}? (y/n)  You can't undo"
+                    f"Are you sure to want to delete {path}? (y/n)  You can't undo"
                     " this action."
                 )
                 delete_in_storage = response == "y"
             else:
                 delete_in_storage = storage
+        if not delete_in_storage:
+            logger.warning(
+                f"you will retain a dangling store here: {path}, not referenced via an artifact"
+            )
         # we don't yet have logic to bring back the deleted metadata record
         # in case storage deletion fails - this is important for ACID down the road
-        if delete_in_storage is True:
-            delete_msg = delete_storage(filepath)
+        if delete_in_storage:
+            delete_msg = delete_storage(path)
             if delete_msg != "did-not-delete":
-                logger.success(f"deleted {colors.yellow(f'{filepath}')}")
+                logger.success(f"deleted {colors.yellow(f'{path}')}")
 
 
 def _delete_skip_storage(artifact, *args, **kwargs) -> None:

lamindb/_query_set.py

@@ -286,9 +286,11 @@ class QuerySet(models.QuerySet, CanValidate, IsTree):
 
 
 def filter_query_set_by_latest_version(ordered_query_set: QuerySet) -> RecordsList:
-    if len(ordered_query_set) == 0:
+    # evaluating length can be very costly, hence, the try-except block
+    try:
+        first_record = ordered_query_set[0]
+    except IndexError:
         return ordered_query_set
-    first_record = ordered_query_set[0]
     records_in_view = {}
     records_in_view[first_record.stem_uid] = first_record
     for record in ordered_query_set:

lamindb/_save.py

@@ -171,7 +171,7 @@ def copy_or_move_to_cache(artifact: Artifact, storage_path: UPath):
     cache_dir = settings._storage_settings.cache_dir
 
     # just delete from the cache dir if a local instance
-    if not lamindb_setup.settings.storage.is_cloud:
+    if not lamindb_setup.settings.storage.type_is_cloud:
         if cache_dir in local_path.parents:
             if is_dir:
                 shutil.rmtree(local_path)

lamindb/core/_data.py

@@ -357,7 +357,12 @@ def _track_run_input(
     is_run_input: bool | None = None,
     run: Run | None = None,
 ):
-    if run is None:
+    # this is an internal hack right now for project-flow, but we can allow this
+    # for the user in the future
+    if isinstance(is_run_input, Run):
+        run = is_run_input
+        is_run_input = True
+    elif run is None:
         run = run_context.run
     # consider that data is an iterable of Data
     data_iter: Iterable[Data] = [data] if isinstance(data, Data) else data

lamindb/integrations/_vitessce.py

@@ -39,15 +39,15 @@ def save_vitessce_config(vitessce_config: VitessceConfig, description: str) -> A
             if "url" not in file:
                 raise ValueError("Each file must have a 'url' key.")
             filename = file["url"].split("/")[-1]
-            assert filename.endswith((".anndata.zarr", ".spatialdata.zarr", ".zarr"))
-            filestem = (
-                filename.replace(".anndata.zarr", "")
-                .replace(".spatialdata.zarr", "")
-                .replace(".zarr", "")
+            assert filename.endswith((".anndata.zarr", ".spatialdata.zarr"))
+            filestem = filename.replace(".anndata.zarr", "").replace(
+                ".spatialdata.zarr", ""
             )
             artifact = Artifact.filter(uid__startswith=filestem).one_or_none()
             if artifact is None:
-                logger.warning(f"could not find dataset in lamindb: {dataset}")
+                logger.warning(
+                    f"could not find dataset '{filestem}' in lamindb: {dataset}"
+                )
             else:
                 input_artifacts.append(artifact)
     # link inputs

{lamindb-0.70.3.dist-info → lamindb-0.70.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.70.3
+Version: 0.70.4
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,8 +9,8 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.65.1
-Requires-Dist: lamindb_setup==0.69.5
+Requires-Dist: lnschema_core==0.65.2
+Requires-Dist: lamindb_setup==0.70.0
 Requires-Dist: lamin_utils==0.13.2
 Requires-Dist: lamin_cli==0.12.3
 Requires-Dist: rapidfuzz
@@ -38,7 +38,7 @@ Requires-Dist: pytest-cov ; extra == "dev"
 Requires-Dist: nbproject_test>=0.5.1 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
-Requires-Dist: readfcs>=1.1.7 ; extra == "fcs"
+Requires-Dist: readfcs>=1.1.8 ; extra == "fcs"
 Requires-Dist: fsspec[gs]==2023.12.2 ; extra == "gcp"
 Requires-Dist: nbproject==0.10.0 ; extra == "jupyter"
 Requires-Dist: nbstripout==0.6.1 ; extra == "jupyter"

{lamindb-0.70.3.dist-info → lamindb-0.70.4.dist-info}/RECORD

@@ -1,6 +1,6 @@
-lamindb/__init__.py,sha256=y-amn3quDKMdf4F0-sm8mFvjJButkNAFPpwMP5bZNj0,2163
-lamindb/_annotate.py,sha256=B0KSvo5S2kJPeMMqy2SSFkqRJCS2QRC4NtI0_vWEZMs,43080
-lamindb/_artifact.py,sha256=SoGsS-uZP7AdrlEWuMC0l50EkNYSvtzrEiXzU5R6NWY,38557
+lamindb/__init__.py,sha256=n_WJSqcrctVzdr83pL8gZ--FI9vu1ZoqTL5AXEdq8LA,2163
+lamindb/_annotate.py,sha256=kgbilILfgzoS-GEpjxzVwRMs7CoSa9BNEcIWXFBW69I,43915
+lamindb/_artifact.py,sha256=E104JM5_Brw7BxJLBTE0acl7Oz7j5R7pPgVgrbHz79I,39279
 lamindb/_can_validate.py,sha256=nvoZG-35n3HofkY4Xc6hBv9AV54_RDan7Hzp5TuqY9I,14709
 lamindb/_collection.py,sha256=SDM35R_5WHrgLKjVb14Q8-Rz_gn5hdZLJobPcanm4PM,14627
 lamindb/_feature.py,sha256=srAKchY7gqD-h-cWlEiAWuHlpFKFwv0PWIA-JX0Go8c,6758
@@ -11,17 +11,17 @@ lamindb/_from_values.py,sha256=DVXjnQ2wwNw-2bFzy0uXLdVlqoprrn95hTnrXwn-KqM,12638
 lamindb/_is_versioned.py,sha256=0PgRCmxEmYDcAjllLSOYZm132B1lW6QgmBBERhRyFt0,1341
 lamindb/_parents.py,sha256=N9T8jbd3eaoHDLE9TD1y1QgGcO81E6Brapy8LILzRCQ,14790
 lamindb/_query_manager.py,sha256=3zokXqxgj9vTJBnN2sbYKS-q69fyDDPF_aGq_rFHzXU,4066
-lamindb/_query_set.py,sha256=fy6xMK9MPGbD8D_i5iNzR8XA009W05ud4tbgrzd5-Vg,11287
+lamindb/_query_set.py,sha256=K_0rJ6Keltl3Pvglvd7kkzkJEy2u6Kp0TKiHLzwqH18,11359
 lamindb/_registry.py,sha256=-Bv10zSr6IY7QM5pu_35NiVjQDJnBcXRECVe9h7GEuY,19336
 lamindb/_run.py,sha256=b7A52M1On3QzFgIYyfQoz5Kk7V3wcu9p_Prq5bzd8v8,1838
-lamindb/_save.py,sha256=x16FBwltaTd1tnXm_zCxkvuVxyon6vRtekf37CfepXg,11426
+lamindb/_save.py,sha256=C4sPr0slgMmxDdiOcaLhIiHOqW9c3DnIz1uj9NlsnXQ,11431
 lamindb/_storage.py,sha256=VW8xq3VRv58-ciholvOdlcgvp_OIlLxx5GxLt-e2Irs,614
 lamindb/_transform.py,sha256=rxojJ91qQSkeYDHYbwqjFAYxBMgJd3cq_K7Z0n5g8Aw,3482
 lamindb/_ulabel.py,sha256=e5dw9h1tR0_u-DMn7Gzx0WhUhV5w7j4v3QbnLWQV7eI,1941
 lamindb/_utils.py,sha256=LGdiW4k3GClLz65vKAVRkL6Tw-Gkx9DWAdez1jyA5bE,428
 lamindb/_view.py,sha256=GV1FrqIMmdooEkA-5zvcTWgV1nqx1sehi6WdWEaFpxM,2171
 lamindb/core/__init__.py,sha256=MB1gEMKUf0GBQrI3dH8WRZOZQmWR4HIojXK_hXXVdqA,1235
-lamindb/core/_data.py,sha256=En3v29eiJARy5l7nSsttAsDsqDLTZ4-xM8fCNyVzExI,17465
+lamindb/core/_data.py,sha256=xULvge-txEO4r4amNQZRZTH3n3BqOLWauyNfxbB6WOA,17674
 lamindb/core/_feature_manager.py,sha256=LlYgU71AoTnrseWFCq-oZkUAYWITtRR7BNFm0AhHe-c,15773
 lamindb/core/_label_manager.py,sha256=0RtegYnK3zIisOnd970EobOrHMpp7OCH-mEoPrPXw2c,9075
 lamindb/core/_mapped_collection.py,sha256=_OwFZh5SePDUD70XIK5kngv3we_Z5-YdGHNfpUSatSQ,19469
@@ -46,10 +46,10 @@ lamindb/core/storage/_zarr.py,sha256=5ceEz6YIvgvUnVVNWhK5Z4W0WfrvyvY82Yna5jSX1_E
 lamindb/core/storage/objects.py,sha256=5LbBeZVKuOOB8DceSE-PN8elKY0N9OhFXZPQJE4lK48,1538
 lamindb/core/storage/paths.py,sha256=kvu4Xi4dvreXpg4iuskN_nd2yyGmEdCmoIfi3nCrTyo,7728
 lamindb/integrations/__init__.py,sha256=aH2PmO2m4-vwIifMYTB0Fyyr_gZWtVnV71jT0tVWSw0,123
-lamindb/integrations/_vitessce.py,sha256=Ii2YhGwXH_tNDS9MXzxNekthWoDmDGpgGxAOVcTIbB4,2550
+lamindb/integrations/_vitessce.py,sha256=b0FqTBsP-M6Q7xCYXVwFwM8DOIeeOBZEhYbryhtq4gk,2535
 lamindb/setup/__init__.py,sha256=OwZpZzPDv5lPPGXZP7-zK6UdO4FHvvuBh439yZvIp3A,410
 lamindb/setup/core/__init__.py,sha256=SevlVrc2AZWL3uALbE5sopxBnIZPWZ1IB0NBDudiAL8,167
-lamindb-0.70.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-0.70.3.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb-0.70.3.dist-info/METADATA,sha256=CYkk_Pk8Xqup7tDkwgd2qiuibHZqAlMG3we02vXTIV8,2835
-lamindb-0.70.3.dist-info/RECORD,,
+lamindb-0.70.4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-0.70.4.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb-0.70.4.dist-info/METADATA,sha256=tpsQ0FARcje5BTONwg1mer7gucqwICMw1RmApXGME0I,2835
+lamindb-0.70.4.dist-info/RECORD,,