lamindb 1.10.1__py3-none-any.whl → 1.11a1__py3-none-any.whl

lamindb/models/sqlrecord.py

@@ -334,7 +334,7 @@ RECORD_REGISTRY_EXAMPLE = """Example::
         experiment.save()
 
         # `Experiment` refers to the registry, which you can query
-        df = Experiment.filter(name__startswith="my ").df()
+        df = Experiment.filter(name__startswith="my ").to_dataframe()
 """
 
 
@@ -425,7 +425,7 @@ class Registry(ModelBase):
 
         Examples:
             >>> ln.ULabel(name="my label").save()
-            >>> ln.ULabel.filter(name__startswith="my").df()
+            >>> ln.ULabel.filter(name__startswith="my").to_dataframe()
         """
         from .query_set import QuerySet
 
@@ -464,7 +464,7 @@ class Registry(ModelBase):
 
         return QuerySet(model=cls).get(idlike, **expressions)
 
-    def df(
+    def to_dataframe(
         cls,
         include: str | list[str] | None = None,
         features: bool | list[str] | str = False,
@@ -492,21 +492,30 @@ class Registry(ModelBase):
 
             Include the name of the creator in the `DataFrame`:
 
-            >>> ln.ULabel.df(include="created_by__name"])
+            >>> ln.ULabel.to_dataframe(include="created_by__name"])
 
             Include display of features for `Artifact`:
 
-            >>> df = ln.Artifact.df(features=True)
+            >>> df = ln.Artifact.to_dataframe(features=True)
             >>> ln.view(df)  # visualize with type annotations
 
             Only include select features:
 
-            >>> df = ln.Artifact.df(features=["cell_type_by_expert", "cell_type_by_model"])
+            >>> df = ln.Artifact.to_dataframe(features=["cell_type_by_expert", "cell_type_by_model"])
         """
         query_set = cls.filter()
         if hasattr(cls, "updated_at"):
             query_set = query_set.order_by("-updated_at")
-        return query_set[:limit].df(include=include, features=features)
+        return query_set[:limit].to_dataframe(include=include, features=features)
+
+    @deprecated(new_name="to_dataframe")
+    def df(
+        cls,
+        include: str | list[str] | None = None,
+        features: bool | list[str] | str = False,
+        limit: int = 100,
+    ) -> pd.DataFrame:
+        return cls.to_dataframe(include, features, limit)
 
     @doc_args(_search.__doc__)
     def search(
@@ -795,7 +804,7 @@ class BaseSQLRecord(models.Model, metaclass=Registry):
         artifacts: list = []
         if self.__class__.__name__ == "Collection" and self.id is not None:
             # when creating a new collection without being able to access artifacts
-            artifacts = self.ordered_artifacts.list()
+            artifacts = self.ordered_artifacts.to_list()
         pre_existing_record = None
         # consider records that are being transferred from other databases
         transfer_logs: dict[str, list[str]] = {
@@ -920,13 +929,15 @@ class BaseSQLRecord(models.Model, metaclass=Registry):
 
     def delete(self) -> None:
         """Delete."""
-        # note that the logic below does not fire if a record is moved to the trash
-        # the idea is that moving a record to the trash should move its entire version family
-        # to the trash, whereas permanently deleting should default to only deleting a single record
-        # of a version family
-        # we can consider making it easy to permanently delete entire version families as well,
-        # but that's for another time
-        if isinstance(self, IsVersioned) and self.is_latest:
+        # deal with versioned records
+        # _overwrite_versions is set to True for folder artifacts
+        # no need to set the new latest version becase all versions are deleted
+        # when deleting the latest version of a folder artifact
+        if (
+            isinstance(self, IsVersioned)
+            and self.is_latest
+            and not getattr(self, "_overwrite_versions", False)
+        ):
             new_latest = (
                 self.__class__.objects.using(self._state.db)
                 .filter(is_latest=False, uid__startswith=self.stem_uid)
@@ -938,7 +949,7 @@ class BaseSQLRecord(models.Model, metaclass=Registry):
                 with transaction.atomic():
                     new_latest.save()
                     super().delete()  # type: ignore
-                logger.warning(f"new latest version is {new_latest}")
+                logger.warning(f"new latest version is: {new_latest}")
                 return None
         super().delete()
 
@@ -952,6 +963,7 @@ class Space(BaseSQLRecord):
     """
 
     class Meta:
+        app_label = "lamindb"
         constraints = [
             models.UniqueConstraint(Lower("name"), name="unique_space_name_lower")
         ]
@@ -964,8 +976,7 @@ class Space(BaseSQLRecord):
         editable=False,
         unique=True,
         max_length=12,
-        default="aaaaaaaaaaaaa",
-        db_default="aaaaaaaaaaaa",
+        default=base62_12,
         db_index=True,
     )
     """Universal id."""
@@ -998,6 +1009,21 @@ class Space(BaseSQLRecord):
         *args,
         **kwargs,
     ):
+        if "uid" not in kwargs:
+            warn = False
+            msg = ""
+            isettings = setup_settings.instance
+            if (dialect := isettings.dialect) != "postgresql":
+                warn = True
+                msg = f"on {dialect} databases"
+            elif not isettings.is_on_hub:
+                warn = True
+                msg = "on local instances"
+            if warn:
+                logger.warning(
+                    f"creating spaces manually {msg} is possible for demo purposes, "
+                    "but does *not* affect access permissions"
+                )
         super().__init__(*args, **kwargs)
 
 
@@ -1007,6 +1033,12 @@ class Branch(BaseSQLRecord):
     Every `SQLRecord` has a `branch` field, which dictates where a record appears in queries & searches.
     """
 
+    class Meta:
+        app_label = "lamindb"
+        constraints = [
+            models.UniqueConstraint(Lower("name"), name="unique_branch_name_lower")
+        ]
+
     # below isn't fully implemented but a roadmap
     # - 3: template (hidden in queries & searches)
     # - 2: locked (same as default, but locked for edits except for space admins)
@@ -1018,11 +1050,6 @@ class Branch(BaseSQLRecord):
     # that can be merged onto the main branch in an experience akin to a Pull Request. The mapping
     # onto a semantic branch name is handled through LaminHub.
 
-    class Meta:
-        constraints = [
-            models.UniqueConstraint(Lower("name"), name="unique_branch_name_lower")
-        ]
-
     id: int = models.AutoField(primary_key=True)
     """An integer id that's synchronized for a family of coupled database instances.
 
@@ -1119,6 +1146,76 @@ class SQLRecord(BaseSQLRecord, metaclass=Registry):
     def _branch_code(self, value: int):
         self.branch_id = value
 
+    def delete(self, permanent: bool | None = None, **kwargs) -> None:
+        """Delete record.
+
+        Args:
+            permanent: Whether to permanently delete the record (skips trash).
+
+        Examples:
+
+            For any `SQLRecord` object `record`, call:
+
+            >>> record.delete()
+        """
+        if self._state.adding:
+            logger.warning("record is not yet saved, delete has no effect")
+            return
+        name_with_module = self.__class__.__get_name_with_module__()
+
+        if name_with_module == "Artifact":
+            # this first check means an invalid delete fails fast rather than cascading through
+            # database and storage permission errors
+            isettings = setup_settings.instance
+            if self.storage.instance_uid != isettings.uid and (
+                kwargs["storage"] or kwargs["storage"] is None
+            ):
+                from ..errors import IntegrityError
+                from .storage import Storage
+
+                raise IntegrityError(
+                    "Cannot simply delete artifacts outside of this instance's managed storage locations."
+                    "\n(1) If you only want to delete the metadata record in this instance, pass `storage=False`"
+                    f"\n(2) If you want to delete the artifact in storage, please load the managing lamindb instance (uid={self.storage.instance_uid})."
+                    f"\nThese are all managed storage locations of this instance:\n{Storage.filter(instance_uid=isettings.uid).to_dataframe()}"
+                )
+
+        # change branch_id to trash
+        trash_branch_id = -1
+        if self.branch_id > trash_branch_id and permanent is not True:
+            self.branch_id = trash_branch_id
+            self.save()
+            logger.warning(f"moved record to trash (`branch_id = -1`): {self}")
+            return
+
+        # permanent delete
+        if permanent is None:
+            response = input(
+                f"Record {self.uid} is already in trash! Are you sure you want to delete it from your"
+                " database? You can't undo this action. (y/n) "
+            )
+            delete_record = response == "y"
+        else:
+            delete_record = permanent
+
+        if delete_record:
+            if name_with_module == "Run":
+                from .run import delete_run_artifacts
+
+                delete_run_artifacts(self)
+            elif name_with_module == "Transform":
+                from .transform import delete_transform_relations
+
+                delete_transform_relations(self)
+            elif name_with_module == "Artifact":
+                from .artifact import delete_permanently
+
+                delete_permanently(
+                    self, storage=kwargs["storage"], using_key=kwargs["using_key"]
+                )
+            if name_with_module != "Artifact":
+                super().delete()
+
 
 def _format_django_validation_error(record: SQLRecord, e: DjangoValidationError):
     """Pretty print Django validation errors."""
@@ -1464,7 +1561,7 @@ def check_name_change(record: SQLRecord):
                 .exclude(feature_id=None)  # must have a feature
                 .distinct()
             )
-            artifact_ids = linked_records.list("artifact__uid")
+            artifact_ids = linked_records.to_list("artifact__uid")
             n = len(artifact_ids)
             if n > 0:
                 s = "s" if n > 1 else ""
@@ -1482,7 +1579,7 @@ def check_name_change(record: SQLRecord):
         # when a feature is renamed
         elif isinstance(record, Feature):
             # only internal features are associated with schemas
-            linked_artifacts = Artifact.filter(feature_sets__features=record).list(
+            linked_artifacts = Artifact.filter(feature_sets__features=record).to_list(
                 "uid"
             )
             n = len(linked_artifacts)
@@ -1806,6 +1903,7 @@ class Migration(BaseSQLRecord):
 
     class Meta:
         db_table = "django_migrations"
+        app_label = "lamindb"
         managed = False
 
 

lamindb/models/storage.py

@@ -4,6 +4,7 @@ from typing import (
     TYPE_CHECKING,
     overload,
 )
+from uuid import UUID
 
 from django.db import models
 from lamin_utils import logger
@@ -11,6 +12,8 @@ from lamindb_setup import settings as setup_settings
 from lamindb_setup.core._hub_core import (
     delete_storage_record,
     get_storage_records_for_instance,
+    select_space,
+    update_storage_with_space,
 )
 from lamindb_setup.core._settings_storage import (
     StorageSettings,
@@ -25,7 +28,7 @@ from lamindb.base.fields import (
 
 from ..base.ids import base62_12
 from .run import TracksRun, TracksUpdates
-from .sqlrecord import SQLRecord
+from .sqlrecord import Space, SQLRecord
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -61,22 +64,31 @@ class Storage(SQLRecord, TracksRun, TracksUpdates):
 
     .. dropdown:: Managing access to storage locations across instances
 
-        You can manage access through AWS policies that you attach to your S3 bucket
-        or leverage LaminHub's fine-grained access management.
+        You can manage access through LaminHub's fine-grained access management or
+        through AWS policies that you attach to your S3 bucket.
 
-        Head over to `https://lamin.ai/{account}/infrastructure`.
-        By clicking the green button that says "Connect S3 bucket", you enable Lamin to issue federated S3 tokens
-        for a bucket so that your collaborators can access data based on their permissions in LaminHub.
+        To enable access management via LaminHub, head over to `https://lamin.ai/{account}/infrastructure`.
+        By clicking the green button that says "Connect S3 bucket", LaminDB will start connecting through federated S3 tokens
+        so that your collaborators access data based on their permissions in LaminHub.
         :doc:`docs:access` has more details.
 
         .. image:: https://lamin-site-assets.s3.amazonaws.com/.lamindb/ze8hkgVxVptSSZEU0000.png
            :width: 800px
 
+        By default, access permissions to a storage location are governed by the access permissions of its managing instance. If you
+        want to further restrict access to a storage location, you can move it into a space::
+
+            space = ln.Space.get(name="my-space")
+            storage_loc = ln.Storage.get(root="s3://my-storace-location")
+            storage_loc.space = space
+            storage_loc.save()
+
         If you don't want to store data in the cloud, you can use local storage locations: :doc:`faq/keep-artifacts-local`.
 
     Args:
         root: `str` The root path of the storage location, e.g., `"./mydir"`, `"s3://my-bucket"`, `"s3://my-bucket/myfolder"`, `"gs://my-bucket/myfolder"`, `"/nfs/shared/datasets/genomics"`, `"/weka/shared/models/"`, ...
         description: `str | None = None` An optional description.
+        space: `Space | None = None` A space to restrict access permissions to the storage location.
         host: `str | None = None` For local storage locations, pass a globally unique host identifier, e.g. `"my-institute-cluster-1"`, `"my-server-abcd"`, ...
 
     See Also:
@@ -107,18 +119,17 @@ class Storage(SQLRecord, TracksRun, TracksUpdates):
 
             ln.Storage(root="/dir/our-shared-dir", host="our-server-123").save()
 
-        Switch to another storage location::
+        Globally switch to another storage location::
 
             ln.settings.storage = "/dir/our-shared-dir"  # or "s3://our-bucket/our-folder", "gs://our-bucket/our-folder", ...
 
-        If you're operating in `keep-artifacts-local` mode (:doc:`faq/keep-artifacts-local`), you can switch among additional local storage locations::
+        Or if you're operating in `keep-artifacts-local` mode (:doc:`faq/keep-artifacts-local`)::
 
-            ln.Storage(root="/dir/our-other-shared-dir", host="our-server-123").save()  # create
-            ln.settings.local_storage = "/dir/our-other-shared-dir"  # switch
+            ln.settings.local_storage = "/dir/our-other-shared-dir"
 
         View all storage locations used in your LaminDB instance::
 
-            ln.Storage.df()
+            ln.Storage.to_dataframe()
 
     Notes:
 
@@ -146,6 +157,7 @@ class Storage(SQLRecord, TracksRun, TracksUpdates):
 
     class Meta(SQLRecord.Meta, TracksRun.Meta, TracksUpdates.Meta):
         abstract = False
+        app_label = "lamindb"
 
     _name_field: str = "root"
 
@@ -174,6 +186,7 @@ class Storage(SQLRecord, TracksRun, TracksUpdates):
         root: str,
         *,
         description: str | None = None,
+        space: Space | None = None,
         host: str | None = None,
     ): ...
 
@@ -190,6 +203,8 @@ class Storage(SQLRecord, TracksRun, TracksUpdates):
     ):
         if len(args) == len(self._meta.concrete_fields):
             super().__init__(*args)
+            self._old_space = self.space
+            self._old_space_id = self.space_id
             return None
         if args:
             assert len(args) == 1, (  # noqa: S101
@@ -213,17 +228,30 @@ class Storage(SQLRecord, TracksRun, TracksUpdates):
             ).one_or_none()
         else:
             storage_record = Storage.filter(root=kwargs["root"]).one_or_none()
+        space = kwargs.get("space", None)
         if storage_record is not None:
             from .sqlrecord import init_self_from_db
 
             init_self_from_db(self, storage_record)
+            self._old_space = self.space
+            self._old_space_id = self.space_id
             return None
 
         skip_preparation = kwargs.pop("_skip_preparation", False)
         if skip_preparation:
+            assert space is None, "`space` must not be set if _skip_preparation is True"  # noqa: S101
             super().__init__(*args, **kwargs)
             return None
 
+        space_uuid = None
+        if space is not None:
+            hub_space_record = select_space(space.uid)
+            if hub_space_record is None:
+                raise ValueError(
+                    "Please first create a space on the hub: https://docs.lamin.ai/access"
+                )
+            space_uuid = UUID(hub_space_record["id"])
+
         # instance_id won't take effect if
         # - there is no write access
         # - the storage location is already managed by another instance
@@ -232,8 +260,8 @@ class Storage(SQLRecord, TracksRun, TracksUpdates):
             instance_id=setup_settings.instance._id,
             instance_slug=setup_settings.instance.slug,
             register_hub=setup_settings.instance.is_on_hub,
-            prevent_register_hub=not setup_settings.instance.is_on_hub,
             region=kwargs.get("region", None),  # host was renamed to region already
+            space_uuid=space_uuid,
         )
         # ssettings performed validation and normalization of the root path
         kwargs["root"] = ssettings.root_as_str  # noqa: S101
@@ -274,6 +302,8 @@ class Storage(SQLRecord, TracksRun, TracksUpdates):
             f"{managed_message} storage location at {kwargs['root']}{is_managed_by_instance}{hub_message}"
         )
         super().__init__(**kwargs)
+        self._old_space = self.space
+        self._old_space_id = self.space_id
 
     @property
     def host(self) -> str | None:
@@ -296,10 +326,25 @@ class Storage(SQLRecord, TracksRun, TracksUpdates):
         access_token = self._access_token if hasattr(self, "_access_token") else None
         return create_path(self.root, access_token=access_token)
 
-    def delete(self) -> None:
+    def save(self, *args, **kwargs):
+        """Save the storage record."""
+        if hasattr(self, "_old_space") and hasattr(self, "_old_space_id"):
+            if (
+                self._old_space != self.space or self._old_space_id != self.space_id
+            ):  # space_id is automatically handled by field tracker according to Claude
+                update_storage_with_space(
+                    storage_lnid=self.uid, space_lnid=self.space.uid
+                )
+        super().save(*args, **kwargs)
+        return self
+
+    def delete(self) -> None:  # type: ignore
+        # type ignore is there because we don't use a trash here unlike everywhere else
         """Delete the storage location.
 
         This errors in case the storage location is not empty.
+
+        Unlike other `SQLRecord`-based registries, this does *not* move the storage record into the trash.
         """
         from .. import settings
 
@@ -324,4 +369,4 @@ class Storage(SQLRecord, TracksRun, TracksUpdates):
             ssettings._mark_storage_root.unlink(
                 missing_ok=True  # this is totally weird, but needed on Py3.11
             )
-        super().delete()
+        super(SQLRecord, self).delete()

lamindb/models/transform.py

@@ -30,6 +30,22 @@ if TYPE_CHECKING:
     from .ulabel import ULabel
 
 
+def delete_transform_relations(transform: Transform):
+    from .project import TransformProject
+
+    # query all runs and delete their associated report and env artifacts
+    runs = Run.filter(transform=transform)
+    for run in runs:
+        delete_run_artifacts(run)
+    # CASCADE doesn't do the job below because run_id might be protected through run__transform=self
+    # hence, proactively delete the label links
+    qs = TransformProject.filter(transform=transform)
+    if qs.exists():
+        qs.delete()
+    # at this point, all artifacts have been taken care of
+    # and one can now leverage CASCADE delete
+
+
 # does not inherit from TracksRun because the Transform
 # is needed to define a run
 class Transform(SQLRecord, IsVersioned):
@@ -95,6 +111,7 @@ class Transform(SQLRecord, IsVersioned):
 
     class Meta(SQLRecord.Meta, IsVersioned.Meta):
         abstract = False
+        app_label = "lamindb"
         unique_together = ("key", "hash")
 
     _len_stem_uid: int = 12
@@ -315,23 +332,6 @@ class Transform(SQLRecord, IsVersioned):
         """The latest run of this transform."""
         return self.runs.order_by("-started_at").first()
 
-    def delete(self) -> None:
-        """Delete."""
-        from .project import TransformProject
-
-        # query all runs and delete their artifacts
-        runs = Run.filter(transform=self)
-        for run in runs:
-            delete_run_artifacts(run)
-        # CASCADE doesn't do the job below because run_id might be protected through run__transform=self
-        # hence, proactively delete the labels
-        qs = TransformProject.filter(transform=self)
-        if qs.exists():
-            qs.delete()
-        # at this point, all artifacts have been taken care of
-        # we can now leverage CASCADE delete
-        super().delete()
-
     def view_lineage(self, with_successors: bool = False, distance: int = 5):
         """View lineage of transforms.
 

lamindb/models/ulabel.py

@@ -83,11 +83,12 @@ class ULabel(SQLRecord, HasParents, CanCurate, TracksRun, TracksUpdates):
 
         Query an artifact by ulabel:
 
-        >>> ln.Artifact.filter(ulabels=train_split).df()
+        >>> ln.Artifact.filter(ulabels=train_split).to_dataframe()
     """
 
     class Meta(SQLRecord.Meta, TracksRun.Meta, TracksUpdates.Meta):
         abstract = False
+        app_label = "lamindb"
 
     _name_field: str = "name"
 
@@ -221,6 +222,7 @@ class ArtifactULabel(BaseSQLRecord, IsLink, TracksRun):
     class Meta:
         # can have the same label linked to the same artifact if the feature is
         # different
+        app_label = "lamindb"
         unique_together = ("artifact", "ulabel", "feature")
 
 
@@ -230,6 +232,7 @@ class TransformULabel(BaseSQLRecord, IsLink, TracksRun):
     ulabel: ULabel = ForeignKey(ULabel, PROTECT, related_name="links_transform")
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("transform", "ulabel")
 
 
@@ -247,6 +250,7 @@ class RunULabel(BaseSQLRecord, IsLink):
     """Creator of record."""
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("run", "ulabel")
 
 
@@ -263,4 +267,5 @@ class CollectionULabel(BaseSQLRecord, IsLink, TracksRun):
     feature_ref_is_name: bool | None = BooleanField(null=True)
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("collection", "ulabel")

{lamindb-1.10.1.dist-info → lamindb-1.11a1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb
-Version: 1.10.1
+Version: 1.11a1
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10,<3.14
@@ -10,8 +10,8 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: lamin_utils==0.15.0
-Requires-Dist: lamin_cli==1.6.1
-Requires-Dist: lamindb_setup[aws]==1.9.1
+Requires-Dist: lamin_cli==1.7.0
+Requires-Dist: lamindb_setup[aws]==1.10.0
 Requires-Dist: pyyaml
 Requires-Dist: pyarrow
 Requires-Dist: pandera>=0.24.0