lamindb 1.2a2__py3-none-any.whl → 1.3.1__py3-none-any.whl

lamindb/models/artifact.py

@@ -16,6 +16,7 @@ from django.db.models import CASCADE, PROTECT, Q
 from lamin_utils import colors, logger
 from lamindb_setup import settings as setup_settings
 from lamindb_setup._init_instance import register_storage_in_instance
+from lamindb_setup.core import doc_args
 from lamindb_setup.core._settings_storage import init_storage
 from lamindb_setup.core.hashing import HASH_LENGTH, hash_dir, hash_file
 from lamindb_setup.core.types import UPathStr
@@ -37,6 +38,7 @@ from lamindb.errors import FieldValidationError
 from lamindb.models.query_set import QuerySet
 
 from ..base.users import current_user_id
+from ..core._compat import is_package_installed
 from ..core.loaders import load_to_memory
 from ..core.storage import (
     LocalPathClasses,
@@ -48,7 +50,6 @@ from ..core.storage import (
 from ..core.storage._anndata_accessor import _anndata_n_observations
 from ..core.storage._pyarrow_dataset import PYARROW_SUFFIXES
 from ..core.storage._tiledbsoma import _soma_n_observations
-from ..core.storage.objects import is_package_installed
 from ..core.storage.paths import (
     AUTO_KEY_PREFIX,
     auto_storage_key_from_artifact,
@@ -93,6 +94,8 @@ WARNING_RUN_TRANSFORM = "no run & transform got linked, call `ln.track()` & re-r
 
 WARNING_NO_INPUT = "run input wasn't tracked, call `ln.track()` and re-run"
 
+DEBUG_KWARGS_DOC = "**kwargs: Internal arguments for debugging."
+
 try:
     from ..core.storage._zarr import identify_zarr_type
 except ImportError:
@@ -113,6 +116,7 @@ if TYPE_CHECKING:
 
     from lamindb.base.types import StrField
     from lamindb.core.storage._backed_access import AnnDataAccessor, BackedAccessor
+    from lamindb.core.types import ScverseDataStructures
 
     from ..base.types import (
         ArtifactKind,
@@ -126,7 +130,7 @@ if TYPE_CHECKING:
 INCONSISTENT_STATE_MSG = (
     "Trying to read a folder artifact from an outdated version, "
     "this can result in an incosistent state.\n"
-    "Read from the latest version: artifact.versions.filter(is_latest=True).one()"
+    "Read from the latest version: artifact.versions.get(is_latest=True)"
 )
 
 
@@ -364,7 +368,7 @@ def get_relative_path_to_directory(
 
 def get_artifact_kwargs_from_data(
     *,
-    data: Path | UPath | str | pd.DataFrame | AnnData | MuData,
+    data: Path | UPath | str | pd.DataFrame | ScverseDataStructures,
     key: str | None,
     run: Run | None,
     format: str | None,
@@ -554,7 +558,7 @@ def data_is_spatialdata(data: SpatialData | UPathStr) -> bool:
 
 
 def _check_otype_artifact(
-    data: UPathStr | pd.DataFrame | AnnData | MuData | SpatialData,
+    data: UPathStr | pd.DataFrame | ScverseDataStructures,
     otype: str | None = None,
 ) -> str:
     if otype is None:
@@ -601,10 +605,10 @@ def get_run(run: Run | None) -> Run | None:
             run = context.run
         if run is None and not settings.creation.artifact_silence_missing_run_warning:
             # here we check that this is not a read-only connection
-            # normally for our connection strings the read-only role name has _read in it
+            # normally for our connection strings the read-only role name has "read" in it
             # not absolutely safe but the worst case is that the warning is not shown
             instance = setup_settings.instance
-            if instance.dialect != "postgresql" or "_read" not in instance.db:
+            if instance.dialect != "postgresql" or "read" not in instance.db:
                 logger.warning(WARNING_RUN_TRANSFORM)
     # suppress run by passing False
     elif not run:
@@ -1427,7 +1431,11 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
             kwargs["uid"] = uid
 
         # only set key now so that we don't do a look-up on it in case revises is passed
-        if revises is not None:
+        if revises is not None and revises.key is not None:
+            assert revises.key.endswith(kwargs["suffix"]), (  # noqa: S101
+                revises.key,
+                kwargs["suffix"],
+            )
             kwargs["key"] = revises.key
 
         kwargs["kind"] = kind
@@ -1467,39 +1475,23 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
     def n_objects(self) -> int:
         return self.n_files
 
-    # add the below because this is what people will have in their code
-    # if they implement the recommended migration strategy
-    # - FeatureSet -> Schema
-    # - featureset -> schema
-    # - feature_set -> schema
-    # @property
-    # def schemas(self) -> QuerySet[Schema]:
-    #     """Schemas linked to artifact via many-to-many relationship.
-
-    #     Is now mediating the private `.feature_sets` relationship during
-    #     a transition period to better schema management.
-
-    #     .. versionchanged: 1.0
-    #        Was previously called `.feature_sets`.
-
-    #     """
-    #     return self.feature_sets
-
     @property
     def path(self) -> Path:
         """Path.
 
-        File in cloud storage, here AWS S3:
+        Example::
 
-        >>> artifact = ln.Artifact("s3://my-bucket/my-file.csv").save()
-        >>> artifact.path
-        S3QueryPath('s3://my-bucket/my-file.csv')
+            import lamindb as ln
 
-        File in local storage:
+            # File in cloud storage, here AWS S3:
+            artifact = ln.Artifact("s3://my-bucket/my-file.csv").save()
+            artifact.path
+            #S3QueryPath('s3://my-bucket/my-file.csv')
 
-        >>> ln.Artifact("./myfile.csv", key="myfile.csv").save()
-        >>> artifact.path
-        PosixPath('/home/runner/work/lamindb/lamindb/docs/guide/mydata/myfile.csv')
+            # File in local storage:
+            ln.Artifact("./myfile.csv", key="myfile.csv").save()
+            artifact.path
+            #> PosixPath('/home/runner/work/lamindb/lamindb/docs/guide/mydata/myfile.csv')
         """
         from lamindb import settings
 
@@ -1519,6 +1511,34 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
             filepath, cache_key=cache_key
         )
 
+    @classmethod
+    def get(
+        cls,
+        idlike: int | str | None = None,
+        **expressions,
+    ) -> Artifact:
+        """Get a single artifact.
+
+        Args:
+            idlike: Either a uid stub, uid or an integer id.
+            expressions: Fields and values passed as Django query expressions.
+
+        Raises:
+            :exc:`docs:lamindb.errors.DoesNotExist`: In case no matching record is found.
+
+        See Also:
+            - Guide: :doc:`docs:registries`
+            - Method in `Record` base class: :meth:`~lamindb.models.Record.get`
+
+        Examples::
+
+            artifact = ln.Artifact.get("tCUkRcaEjTjhtozp0000")
+            artifact = ln.Arfifact.get(key="my_datasets/my_file.parquet")
+        """
+        from .query_set import QuerySet
+
+        return QuerySet(model=cls).get(idlike, **expressions)
+
     @classmethod
     def from_df(
         cls,
@@ -1546,17 +1566,19 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
             :class:`~lamindb.Feature`
                 Track features.
 
-        Examples:
-            >>> df = ln.core.datasets.df_iris_in_meter_batch1()
-            >>> df.head()
-              sepal_length sepal_width petal_length petal_width iris_organism_code
-            0        0.051       0.035        0.014       0.002                 0
-            1        0.049       0.030        0.014       0.002                 0
-            2        0.047       0.032        0.013       0.002                 0
-            3        0.046       0.031        0.015       0.002                 0
-            4        0.050       0.036        0.014       0.002                 0
-            >>> artifact = ln.Artifact.from_df(df, description="Iris flower collection batch1")
-            >>> artifact.save()
+        Example::
+
+            import lamindb as ln
+
+            df = ln.core.datasets.df_iris_in_meter_batch1()
+            df.head()
+            #>   sepal_length sepal_width petal_length petal_width iris_organism_code
+            #> 0        0.051       0.035        0.014       0.002                 0
+            #> 1        0.049       0.030        0.014       0.002                 0
+            #> 2        0.047       0.032        0.013       0.002                 0
+            #> 3        0.046       0.031        0.015       0.002                 0
+            #> 4        0.050       0.036        0.014       0.002                 0
+            artifact = ln.Artifact.from_df(df, key="iris/result_batch1.parquet").save()
         """
         artifact = Artifact(  # type: ignore
             data=df,
@@ -1599,12 +1621,12 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
             :class:`~lamindb.Feature`
                 Track features.
 
-        Examples:
-            >>> import bionty as bt
-            >>> bt.settings.organism = "human"
-            >>> adata = ln.core.datasets.anndata_with_obs()
-            >>> artifact = ln.Artifact.from_anndata(adata, description="mini anndata with obs")
-            >>> artifact.save()
+        Example::
+
+            import lamindb as ln
+
+            adata = ln.core.datasets.anndata_with_obs()
+            artifact = ln.Artifact.from_anndata(adata, key="mini_anndata_with_obs.h5ad").save()
         """
         if not data_is_anndata(adata):
             raise ValueError(
@@ -1661,12 +1683,12 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
             :class:`~lamindb.Feature`
                 Track features.
 
-        Examples:
-            >>> import bionty as bt
-            >>> bt.settings.organism = "human"
-            >>> mdata = ln.core.datasets.mudata_papalexi21_subset()
-            >>> artifact = ln.Artifact.from_mudata(mdata, description="a mudata object")
-            >>> artifact.save()
+        Example::
+
+            import lamindb as ln
+
+            mdata = ln.core.datasets.mudata_papalexi21_subset()
+            artifact = ln.Artifact.from_mudata(mdata, key="mudata_papalexi21_subset.h5mu").save()
         """
         if not data_is_mudata(mdata):
             raise ValueError("data has to be a MuData object or a path to MuData-like")
@@ -1711,8 +1733,11 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
             :class:`~lamindb.Feature`
                 Track features.
 
-        Examples:
-            >>> artifact = ln.Artifact.from_spatialdata(sdata, key="my_dataset.zarr")
+        Example::
+
+            import lamindb as ln
+
+            artifact = ln.Artifact.from_spatialdata(sdata, key="my_dataset.zarr").save()
         """
         if not data_is_spatialdata(sdata):
             raise ValueError(
@@ -1753,9 +1778,11 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
             revises: An old version of the artifact.
             run: The run that creates the artifact.
 
-        Examples:
-            >>> artifact = ln.Artifact.from_tiledbsoma("s3://mybucket/store.tiledbsoma", description="a tiledbsoma store")
-            >>> artifact.save()
+        Example::
+
+            import lamindb as ln
+
+            artifact = ln.Artifact.from_tiledbsoma("s3://mybucket/store.tiledbsoma", description="a tiledbsoma store").save()
         """
         if UPath(path).suffix != ".tiledbsoma":
             raise ValueError(
@@ -1797,10 +1824,13 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
                 of a registered storage location, the inferred key defaults to `path.name`.
             run: A `Run` object.
 
-        Examples:
-            >>> dir_path = ln.core.datasets.generate_cell_ranger_files("sample_001", ln.settings.storage)
-            >>> artifacts = ln.Artifact.from_dir(dir_path)
-            >>> ln.save(artifacts)
+        Example::
+
+            import lamindb as ln
+
+            dir_path = ln.core.datasets.generate_cell_ranger_files("sample_001", ln.settings.storage)
+            artifacts = ln.Artifact.from_dir(dir_path)
+            ln.save(artifacts)
         """
         from lamindb import settings
 
@@ -1987,6 +2017,10 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
         # no need to upload if new file is already in storage
         self._to_store = not check_path_in_storage
 
+        # update old suffix with the new one so that checks in record pass
+        # replace() supports changing the suffix
+        self._old_suffix = self.suffix
+
     def open(
         self, mode: str = "r", is_run_input: bool | None = None, **kwargs
     ) -> Union[
@@ -2005,19 +2039,24 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
         Args:
             mode: can only be `"w"` (write mode) for `tiledbsoma` stores,
                 otherwise should be always `"r"` (read-only mode).
+            is_run_input: Whether to track this artifact as run input.
+            **kwargs: Keyword arguments for the accessor, i.e. `h5py` or `zarr` connection,
+                `pyarrow.dataset.dataset`.
 
         Notes:
             For more info, see tutorial: :doc:`/arrays`.
 
-        Examples:
+        Example::
+
+            import lamindb as ln
 
-            Read AnnData in backed mode from cloud:
+            # Read AnnData in backed mode from cloud
 
-            >>> artifact = ln.Artifact.get(key="lndb-storage/pbmc68k.h5ad")
-            >>> artifact.open()
-            AnnDataAccessor object with n_obs × n_vars = 70 × 765
-                constructed for the AnnData object pbmc68k.h5ad
-                ...
+            artifact = ln.Artifact.get(key="lndb-storage/pbmc68k.h5ad")
+            artifact.open()
+            #> AnnDataAccessor object with n_obs × n_vars = 70 × 765
+            #>     constructed for the AnnData object pbmc68k.h5ad
+            #>     ...
         """
         if self._overwrite_versions and not self.is_latest:
             raise ValueError(INCONSISTENT_STATE_MSG)
@@ -2118,11 +2157,18 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
         _track_run_input(self, is_run_input)
         return access
 
-    def load(self, is_run_input: bool | None = None, **kwargs) -> Any:
+    def load(
+        self, *, is_run_input: bool | None = None, mute: bool = False, **kwargs
+    ) -> Any:
         """Cache and load into memory.
 
         See all :mod:`~lamindb.core.loaders`.
 
+        Args:
+            is_run_input: Whether to track this artifact as run input.
+            mute: Silence logging of caching progress.
+            **kwargs: Keyword arguments for the loader.
+
         Examples:
 
             Load a `DataFrame`-like artifact:
@@ -2156,7 +2202,9 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
             filepath, cache_key = filepath_cache_key_from_artifact(
                 self, using_key=settings._using_key
             )
-            cache_path = _synchronize_cleanup_on_error(filepath, cache_key=cache_key)
+            cache_path = _synchronize_cleanup_on_error(
+                filepath, cache_key=cache_key, print_progress=not mute
+            )
             try:
                 # cache_path is local so doesn't trigger any sync in load_to_memory
                 access_memory = load_to_memory(cache_path, **kwargs)
@@ -2177,26 +2225,33 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
                     cache_path.unlink(missing_ok=True)
                 # download again and try to load into memory
                 cache_path = _synchronize_cleanup_on_error(
-                    filepath, cache_key=cache_key
+                    filepath, cache_key=cache_key, print_progress=not mute
                 )
                 access_memory = load_to_memory(cache_path, **kwargs)
         # only call if load is successfull
         _track_run_input(self, is_run_input)
         return access_memory
 
-    def cache(self, is_run_input: bool | None = None) -> Path:
+    @doc_args(DEBUG_KWARGS_DOC)
+    def cache(
+        self, *, is_run_input: bool | None = None, mute: bool = False, **kwargs
+    ) -> Path:
         """Download cloud artifact to local cache.
 
         Follows synching logic: only caches an artifact if it's outdated in the local cache.
 
         Returns a path to a locally cached on-disk object (say a `.jpg` file).
 
-        Examples:
+        Args:
+            mute: Silence logging of caching progress.
+            is_run_input: Whether to track this artifact as run input.
+            {}
 
-            Sync file from cloud and return the local path of the cache:
+        Example::
 
-            >>> artifact.cache()
-            PosixPath('/home/runner/work/Caches/lamindb/lamindb-ci/lndb-storage/pbmc68k.h5ad')
+            # Sync file from cloud and return the local path of the cache
+            artifact.cache()
+            #> PosixPath('/home/runner/work/Caches/lamindb/lamindb-ci/lndb-storage/pbmc68k.h5ad')
         """
         from lamindb import settings
 
@@ -2206,7 +2261,11 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
         filepath, cache_key = filepath_cache_key_from_artifact(
             self, using_key=settings._using_key
         )
-        cache_path = _synchronize_cleanup_on_error(filepath, cache_key=cache_key)
+        if mute:
+            kwargs["print_progress"] = False
+        cache_path = _synchronize_cleanup_on_error(
+            filepath, cache_key=cache_key, **kwargs
+        )
         # only call if sync is successfull
         _track_run_input(self, is_run_input)
         return cache_path
@@ -2231,18 +2290,19 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
             permanent: Permanently delete the artifact (skip trash).
             storage: Indicate whether you want to delete the artifact in storage.
 
-        Examples:
+        Example::
 
-            For an `Artifact` object `artifact`, call:
+            import lamindb as ln
 
-            >>> artifact = ln.Artifact.filter(key="some.csv").one()
-            >>> artifact.delete() # delete a single file artifact
+            # For an `Artifact` object `artifact`, call:
+            artifact = ln.Artifact.get(key="some.csv")
+            artifact.delete() # delete a single file artifact
 
-            >>> artifact = ln.Artifact.filter(key="some.tiledbsoma". is_latest=False).first()
-            >>> artiact.delete() # delete an old version, the data will not be deleted
+            artifact = ln.Artifact.filter(key="some.tiledbsoma". is_latest=False).first()
+            artiact.delete() # delete an old version, the data will not be deleted
 
-            >>> artifact = ln.Artifact.filter(key="some.tiledbsoma". is_latest=True).one()
-            >>> artiact.delete() # delete all versions, the data will be deleted or prompted for deletion.
+            artifact = ln.Artifact.get(key="some.tiledbsoma". is_latest=True)
+            artiact.delete() # delete all versions, the data will be deleted or prompted for deletion.
         """
         # this first check means an invalid delete fails fast rather than cascading through
         # database and storage permission errors
@@ -2330,15 +2390,19 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
                 if delete_msg != "did-not-delete":
                     logger.success(f"deleted {colors.yellow(f'{path}')}")
 
+    @doc_args(DEBUG_KWARGS_DOC)
     def save(self, upload: bool | None = None, **kwargs) -> Artifact:
         """Save to database & storage.
 
         Args:
             upload: Trigger upload to cloud storage in instances with hybrid storage mode.
+            {}
 
-        Examples:
-            >>> artifact = ln.Artifact("./myfile.csv", description="myfile")
-            >>> artifact.save()
+        Example::
+
+            import lamindb as ln
+
+            artifact = ln.Artifact("./myfile.csv", key="myfile.parquet").save()
         """
         state_was_adding = self._state.adding
         print_progress = kwargs.pop("print_progress", True)
@@ -2407,8 +2471,9 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
     def restore(self) -> None:
         """Restore from trash.
 
-        Examples:
-            >>> artifact.restore()
+        Example::
+
+            artifact.restore()
         """
         self._branch_code = 1
         self.save()
@@ -2416,8 +2481,9 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
     def describe(self) -> None:
         """Describe relations of record.
 
-        Examples:
-            >>> artifact.describe()
+        Example::
+
+            artifact.describe()
         """
         return describe_artifact_collection(self)
 
@@ -2427,11 +2493,12 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
 
 # can't really just call .cache in .load because of double tracking
 def _synchronize_cleanup_on_error(
-    filepath: UPath, cache_key: str | None = None
+    filepath: UPath, cache_key: str | None = None, **kwargs
 ) -> UPath:
     try:
+        print_progress = kwargs.pop("print_progress", True)
         cache_path = setup_settings.paths.cloud_to_local(
-            filepath, cache_key=cache_key, print_progress=True
+            filepath, cache_key=cache_key, print_progress=print_progress, **kwargs
         )
     except Exception as e:
         if not isinstance(filepath, LocalPathClasses):
@@ -2477,8 +2544,9 @@ class ArtifactParamValue(BasicRecord, LinkORM, TracksRun):
 
 
 def _track_run_input(
-    data: Artifact
-    | Iterable[Artifact],  # can also be Collection | Iterable[Collection]
+    data: (
+        Artifact | Iterable[Artifact]
+    ),  # can also be Collection | Iterable[Collection]
     is_run_input: bool | Run | None = None,
     run: Run | None = None,
 ):
@@ -2542,10 +2610,10 @@ def _track_run_input(
         if run is None:
             if settings.track_run_inputs:
                 # here we check that this is not a read-only connection
-                # normally for our connection strings the read-only role name has _read in it
+                # normally for our connection strings the read-only role name has "read" in it
                 # not absolutely safe but the worst case is that the warning is not shown
                 instance = setup_settings.instance
-                if instance.dialect != "postgresql" or "_read" not in instance.db:
+                if instance.dialect != "postgresql" or "read" not in instance.db:
                     logger.warning(WARNING_NO_INPUT)
         # assume we have a run record
         else: