lamindb 1.11.3__py3-none-any.whl → 1.12.0__py3-none-any.whl

lamindb/models/sqlrecord.py

@@ -46,10 +46,12 @@ from lamindb_setup.core.upath import extract_suffix_from_path
 from lamindb.base import deprecated
 
 from ..base.fields import (
+    BooleanField,
     CharField,
     DateTimeField,
     ForeignKey,
     JSONField,
+    TextField,
 )
 from ..base.ids import base62_12
 from ..base.types import FieldAttr, StrField
@@ -69,6 +71,7 @@ if TYPE_CHECKING:
     import pandas as pd
 
     from .artifact import Artifact
+    from .blocks import Block
     from .run import Run, User
     from .transform import Transform
 
@@ -113,7 +116,6 @@ IPYTHON = getattr(builtins, "__IPYTHON__", False)
 #     "a dance of words, where clarity meets brevity. Every syllable counts,"
 #     "illustrating the skill in compact expression, ensuring the essence of the"
 #     "message shines through within the exacting limit."
-# This is a good maximal length for a description field.
 
 
 class IsLink:
@@ -124,6 +126,14 @@ def deferred_attribute__repr__(self):
     return f"FieldAttr({self.field.model.__name__}.{self.field.name})"
 
 
+def unique_constraint_error_in_error_message(error_msg: str) -> bool:
+    """Check if the error message indicates a unique constraint violation."""
+    return (
+        "UNIQUE constraint failed" in error_msg  # SQLite
+        or "duplicate key value violates unique constraint" in error_msg  # Postgre
+    )
+
+
 FieldAttr.__repr__ = deferred_attribute__repr__  # type: ignore
 
 
@@ -887,10 +897,7 @@ class BaseSQLRecord(models.Model, metaclass=Registry):
                     self.__class__.__name__ in {"Transform", "Artifact"}
                     and isinstance(e, IntegrityError)
                     and "hash" in error_msg
-                    and (
-                        "UNIQUE constraint failed" in error_msg
-                        or "duplicate key value violates unique constraint" in error_msg
-                    )
+                    and unique_constraint_error_in_error_message(error_msg)
                 ):
                     pre_existing_record = self.__class__.get(hash=self.hash)
                     logger.warning(
@@ -900,16 +907,53 @@ class BaseSQLRecord(models.Model, metaclass=Registry):
                 elif (
                     self.__class__.__name__ == "Storage"
                     and isinstance(e, IntegrityError)
-                    and "root" in error_msg
-                    or "uid" in error_msg
+                    and ("root" in error_msg or "uid" in error_msg)
+                    and unique_constraint_error_in_error_message(error_msg)
+                ):
+                    # even if uid was in the error message, we can retrieve based on
+                    # the root because it's going to be the same root
+                    pre_existing_record = self.__class__.get(root=self.root)
+                    init_self_from_db(self, pre_existing_record)
+                elif (
+                    isinstance(e, IntegrityError)
+                    and ("ontology_id" in error_msg or "uid" in error_msg)
                     and (
                         "UNIQUE constraint failed" in error_msg
                         or "duplicate key value violates unique constraint" in error_msg
                     )
                 ):
-                    # even if uid was in the error message, we can retrieve based on
-                    # the root because it's going to be the same root
-                    pre_existing_record = self.__class__.get(root=self.root)
+                    if "UNIQUE constraint failed" in error_msg:  # sqlite
+                        constraint_fields = [
+                            f.split(".")[-1]
+                            for f in error_msg.removeprefix(
+                                "UNIQUE constraint failed: "
+                            ).split(", ")
+                        ]
+                    else:  # postgres
+                        constraint_fields = [
+                            error_msg.split('"')[1]
+                            .split('"')[0]
+                            .removesuffix("_key")
+                            .split("_")[-1]  # field name
+                        ]
+                    # here we query against the all branches with .objects
+                    pre_existing_record = self.__class__.objects.get(
+                        **{f: getattr(self, f) for f in constraint_fields}
+                    )
+                    if pre_existing_record.branch_id == 1:
+                        logger.warning(
+                            f"returning existing {self.__class__.__name__} record with same {', '.join(constraint_fields)}: '{', '.join([str(getattr(self, f)) for f in constraint_fields])}'"
+                        )
+                    else:
+                        # modifies the fields of the existing record with new values of self
+                        # TODO: parents should be properly dealt with
+                        self._parents: list = []
+                        field_names = [i.name for i in self.__class__._meta.fields]
+                        update_attributes(
+                            pre_existing_record,
+                            {f: getattr(self, f) for f in field_names},
+                        )
+                        pre_existing_record.save()
                     init_self_from_db(self, pre_existing_record)
                 elif (
                     isinstance(e, ProgrammingError)
@@ -917,7 +961,7 @@ class BaseSQLRecord(models.Model, metaclass=Registry):
                     and "new row violates row-level security policy" in error_msg
                 ):
                     raise NoWriteAccess(
-                        f"You’re not allowed to write to the space '{self.space.name}'.\n"
+                        f"You're not allowed to write to the space '{self.space.name}'.\n"
                         "Please contact administrators of the space if you need write access."
                     ) from None
                 else:
@@ -964,8 +1008,18 @@ class BaseSQLRecord(models.Model, metaclass=Registry):
                 self.projects.add(ln.context.project)
         return self
 
-    def delete(self) -> None:
-        """Delete."""
+    def delete(self, permanent: bool | None = None) -> None:
+        """Delete.
+
+        Args:
+            permanent: For consistency, `False` raises an error, as soft delete is impossible.
+        """
+        if permanent is False:
+            raise ValueError(
+                f"Soft delete is not possible for {self.__class__.__name__}, "
+                "use 'permanent=True' or 'permanent=None' for permanent deletion."
+            )
+
         delete_record(self, is_soft=False)
 
 
@@ -995,7 +1049,7 @@ class Space(BaseSQLRecord):
         db_index=True,
     )
     """Universal id."""
-    description: str | None = CharField(null=True)
+    description: str | None = TextField(null=True)
     """Description of space."""
     created_at: datetime = DateTimeField(
         editable=False, db_default=models.functions.Now(), db_index=True
@@ -1005,6 +1059,8 @@ class Space(BaseSQLRecord):
         "User", CASCADE, default=None, related_name="+", null=True
     )
     """Creator of space."""
+    blocks: Block
+    """Blocks that annotate this space."""
 
     @overload
     def __init__(
@@ -1083,7 +1139,9 @@ class Branch(BaseSQLRecord):
 
     This id is useful if one wants to apply the same patch to many database instances.
     """
-    description: str | None = CharField(null=True)
+    space: Space = ForeignKey(Space, PROTECT, default=1, db_default=1, related_name="+")
+    """The space associated with the branch."""
+    description: str | None = TextField(null=True)
     """Description of branch."""
     created_at: datetime = DateTimeField(
         editable=False, db_default=models.functions.Now(), db_index=True
@@ -1093,6 +1151,8 @@ class Branch(BaseSQLRecord):
         "User", CASCADE, default=None, related_name="+", null=True
     )
     """Creator of branch."""
+    blocks: Block
+    """Blocks that annotate this branch."""
 
     @overload
     def __init__(
@@ -1134,6 +1194,7 @@ class SQLRecord(BaseSQLRecord, metaclass=Registry):
     machine learning or biological models.
     """
 
+    # we need the db_default when not interacting via django directly on a required field
     branch: Branch = ForeignKey(
         Branch,
         PROTECT,
@@ -1145,6 +1206,8 @@ class SQLRecord(BaseSQLRecord, metaclass=Registry):
     """Whether record is on a branch or in another "special state"."""
     space: Space = ForeignKey(Space, PROTECT, default=1, db_default=1, related_name="+")
     """The space in which the record lives."""
+    is_locked: bool = BooleanField(default=False, db_default=False)
+    """Whether the record is locked for edits."""
     _aux: dict[str, Any] | None = JSONField(default=None, db_default=None, null=True)
     """Auxiliary field for dictionary-like metadata."""
 
@@ -1166,6 +1229,7 @@ class SQLRecord(BaseSQLRecord, metaclass=Registry):
 
         Args:
             permanent: Whether to permanently delete the record (skips trash).
+                If `None`, performs soft delete if the record is not already in the trash.
 
         Examples:
 
@@ -1528,8 +1592,8 @@ def track_current_key_and_name_values(record: SQLRecord):
     # below, we're using __dict__ to avoid triggering the refresh from the database
     # which can lead to a recursion
     if isinstance(record, Artifact):
-        record._old_key = record.__dict__.get("key")
-        record._old_suffix = record.__dict__.get("suffix")
+        record._old_key = record.__dict__.get("key")  # type: ignore
+        record._old_suffix = record.__dict__.get("suffix")  # type: ignore
     elif hasattr(record, "_name_field"):
         record._old_name = record.__dict__.get(record._name_field)
 
@@ -1617,12 +1681,12 @@ def check_key_change(record: Union[Artifact, Transform]):
 
     if not isinstance(record, Artifact) or not hasattr(record, "_old_key"):
         return
-    if record._old_suffix != record.suffix:
+    if record._old_suffix != record.suffix:  # type: ignore
         raise InvalidArgument(
-            f"Changing the `.suffix` of an artifact is not allowed! You tried to change it from '{record._old_suffix}' to '{record.suffix}'."
+            f"Changing the `.suffix` of an artifact is not allowed! You tried to change it from '{record._old_suffix}' to '{record.suffix}'."  # type: ignore
         )
 
-    old_key = record._old_key
+    old_key = record._old_key  # type: ignore
     new_key = record.key
 
     if old_key != new_key:
@@ -1870,6 +1934,9 @@ def record_repr(
     if "created_at" in field_names:
         field_names.remove("created_at")
         field_names.append("created_at")
+    if "is_locked" in field_names:
+        field_names.remove("is_locked")
+        field_names.append("is_locked")
     if field_names[0] != "uid" and "uid" in field_names:
         field_names.remove("uid")
         field_names.insert(0, "uid")
@@ -1891,21 +1958,6 @@ def record_repr(
     return f"{self.__class__.__name__}({fields_joined_str})"
 
 
-# below is code to further format the repr of a record
-#
-# def format_repr(
-#     record: SQLRecord, exclude_field_names: str | list[str] | None = None
-# ) -> str:
-#     if isinstance(exclude_field_names, str):
-#         exclude_field_names = [exclude_field_names]
-#     exclude_field_names_init = ["id", "created_at", "updated_at"]
-#     if exclude_field_names is not None:
-#         exclude_field_names_init += exclude_field_names
-#     return record.__repr__(
-#         include_foreign_keys=False, exclude_field_names=exclude_field_names_init
-#     )
-
-
 SQLRecord.__repr__ = record_repr  # type: ignore
 SQLRecord.__str__ = record_repr  # type: ignore
 

lamindb/models/storage.py

@@ -24,6 +24,7 @@ from lamindb_setup.core.upath import check_storage_is_empty, create_path
 
 from lamindb.base.fields import (
     CharField,
+    TextField,
 )
 
 from ..base.ids import base62_12
@@ -169,8 +170,8 @@ class Storage(SQLRecord, TracksRun, TracksUpdates):
     """Universal id, valid across DB instances."""
     root: str = CharField(db_index=True, unique=True)
     """Root path of storage (cloud or local path)."""
-    description: str | None = CharField(db_index=True, null=True)
-    """A description of what the storage location is used for (optional)."""
+    description: str | None = TextField(null=True)
+    """A description."""
     type: StorageType = CharField(max_length=30, db_index=True)
     """Can be "local" vs. "s3" vs. "gs". Is auto-detected from the format of the `root` path."""
     region: str | None = CharField(max_length=64, db_index=True, null=True)
@@ -338,16 +339,25 @@ class Storage(SQLRecord, TracksRun, TracksUpdates):
         super().save(*args, **kwargs)
         return self
 
-    def delete(self) -> None:  # type: ignore
+    def delete(self, permanent: bool | None = None) -> None:  # type: ignore
         # type ignore is there because we don't use a trash here unlike everywhere else
         """Delete the storage location.
 
         This errors in case the storage location is not empty.
 
         Unlike other `SQLRecord`-based registries, this does *not* move the storage record into the trash.
+
+        Args:
+            permanent: For consistency, `False` raises an error, as soft delete is impossible.
         """
         from .. import settings
 
+        if permanent is False:
+            raise ValueError(
+                "Soft delete is not possible for Storage, "
+                "use 'permanent=True' or 'permanent=None' for permanent deletion."
+            )
+
         assert not self.artifacts.exists(), "Cannot delete storage holding artifacts."  # noqa: S101
         check_storage_is_empty(self.path)
         assert settings.storage.root_as_str != self.root, (  # noqa: S101

lamindb/models/transform.py

@@ -26,6 +26,7 @@ if TYPE_CHECKING:
 
     from lamindb.base.types import TransformType
 
+    from .block import TransformBlock
     from .project import Project, Reference
     from .ulabel import ULabel
 
@@ -127,12 +128,14 @@ class Transform(SQLRecord, IsVersioned):
     # the fact that key is nullable is consistent with Artifact
     # it might turn out that there will never really be a use case for this
     # but there likely also isn't much harm in it except for the mixed type
-    key: str | None = CharField(db_index=True, null=True)
+    # max length for key is 1014 and equals the max lenght of an S3 key & artifact key
+    key: str | None = CharField(db_index=True, null=True, max_length=1024)
     """A name or "/"-separated path-like string.
 
     All transforms with the same key are part of the same version family.
     """
-    description: str | None = CharField(db_index=True, null=True)
+    # db_index on description because sometimes we query for equality in the case of artifacts
+    description: str | None = TextField(null=True, db_index=True)
     """A description."""
     type: TransformType = CharField(
         max_length=20,
@@ -187,6 +190,8 @@ class Transform(SQLRecord, IsVersioned):
         "Transform", PROTECT, related_name="_derived_from", default=None, null=True
     )
     """Creating template."""
+    blocks: TransformBlock
+    """Blocks that annotate this artifact."""
 
     @overload
     def __init__(

lamindb/models/ulabel.py

@@ -11,6 +11,7 @@ from lamindb.base.fields import (
     CharField,
     DateTimeField,
     ForeignKey,
+    TextField,
 )
 from lamindb.errors import FieldValidationError
 
@@ -33,33 +34,15 @@ if TYPE_CHECKING:
 class ULabel(SQLRecord, HasParents, CanCurate, TracksRun, TracksUpdates):
     """Universal labels.
 
+    For new labels, see `Record` instead. Existing labels and code will continue to work
+    but be migrated to the Record registry.
+
     Args:
         name: `str` A name.
         description: `str | None = None` A description.
         reference: `str | None = None` For instance, an external ID or a URL.
         reference_type: `str | None = None` For instance, `"url"`.
 
-    A `ULabel` record provides the easiest way to annotate a dataset
-    with a label: `"My project"`, `"curated"`, or `"Batch X"`:
-
-        >>> my_project = ULabel(name="My project").save()
-        >>> artifact.ulabels.add(my_project)
-
-    Often, a ulabel is measured *within* a dataset. For instance, an artifact
-    might characterize 2 species of the Iris flower (`"setosa"` &
-    `"versicolor"`) measured by a `"species"` feature. Use the
-    :class:`~lamindb.curators.DataFrameCurator` flow to automatically parse, validate, and
-    annotate with labels that are contained in `DataFrame` objects.
-
-    .. note::
-
-        If you work with complex entities like cell lines, cell types, tissues,
-        etc., consider using the pre-defined biological registries in
-        :mod:`bionty` to label artifacts & collections.
-
-        If you work with biological samples, likely, the only sustainable way of
-        tracking metadata, is to create a custom schema module.
-
     See Also:
         :meth:`~lamindb.Feature`
             Dimensions of measurement for artifacts & collections.
@@ -112,8 +95,8 @@ class ULabel(SQLRecord, HasParents, CanCurate, TracksRun, TracksUpdates):
 
     For example, a ulabel "Project" would be a type, and the actual projects "Project 1", "Project 2", would be records of that `type`.
     """
-    description: str | None = CharField(null=True, db_index=True)
-    """A description (optional)."""
+    description: str | None = TextField(null=True)
+    """A description."""
     reference: str | None = CharField(max_length=255, db_index=True, null=True)
     """A simple reference like URL or external ID."""
     reference_type: str | None = CharField(max_length=25, db_index=True, null=True)

{lamindb-1.11.3.dist-info → lamindb-1.12.0.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.3
 Name: lamindb
-Version: 1.11.3
-Summary: A data framework for biology.
+Version: 1.12.0
+Summary: A data lakehouse for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10,<3.14
 Description-Content-Type: text/markdown
@@ -10,22 +10,20 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: lamin_utils==0.15.0
-Requires-Dist: lamin_cli==1.7.2
-Requires-Dist: lamindb_setup[aws]==1.10.2
-Requires-Dist: bionty>=1.7a1
-Requires-Dist: wetlab>=1.5a1
+Requires-Dist: lamin_cli==1.8.0
+Requires-Dist: lamindb_setup[aws]==1.11.0
+Requires-Dist: bionty==1.8.1
+Requires-Dist: wetlab==1.6.1
 Requires-Dist: nbproject==0.11.1
 Requires-Dist: jupytext
 Requires-Dist: nbconvert>=7.2.1
-Requires-Dist: mistune!=3.1.0
 Requires-Dist: pyyaml
 Requires-Dist: pyarrow
 Requires-Dist: pandera>=0.24.0
 Requires-Dist: typing_extensions!=4.6.0
 Requires-Dist: python-dateutil
 Requires-Dist: pandas>=2.0.0
-Requires-Dist: scipy<1.15.0
-Requires-Dist: anndata>=0.8.0,<=0.12.1
+Requires-Dist: anndata>=0.8.0,<=0.12.2
 Requires-Dist: fsspec
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
@@ -60,8 +58,7 @@ Provides-Extra: zarr
 
 # LaminDB - A data lakehouse for biology
 
-LaminDB is an open-source data lakehouse to enable learning at scale in biology.
-It organizes datasets through validation & annotation and provides data lineage, queryability, and reproducibility on top of [FAIR](https://en.wikipedia.org/wiki/FAIR_data) data.
+LaminDB organizes datasets through validation & annotation and provides data lineage, queryability & reproducibility on top of [FAIR](https://en.wikipedia.org/wiki/FAIR_data) data.
 
 <details>
 <summary>Why?</summary>
@@ -80,17 +77,17 @@ Moreover, it provides context through data lineage -- tracing data and code, sci
 **Highlights.**
 
 - **data lineage:** track inputs & outputs of notebooks, scripts, functions & pipelines with a single line of code
-- **unified infrastructure:** access diverse storage locations (local, S3, GCP, ...), SQL databases (Postgres, SQLite) & ontologies
-- **lakehouse capabilities**: manage, monitor & validate features, labels & dataset schemas; perform distributed queries and batch loading
-- **biological data formats:** validate & annotate formats like `DataFrame`, `AnnData`, `MuData`, ... backed by `parquet`, `zarr`, HDF5, LanceDB, DuckDB, ...
-- **biological entities**: organize experimental metadata & extensible ontologies in registries based on the Django ORM
+- **unified access:** storage locations (local, S3, GCP, ...), SQL databases (Postgres, SQLite) & ontologies
+- **lakehouse**: manage, monitor & validate features, labels & dataset schemas; distributed queries and batch loading
+- **biological formats:** validate & annotate `DataFrame`, `AnnData`, `SpatialData`, ... backed by `parquet`, `zarr`, HDF5, LanceDB, ...
+- **biological entities**: manage experimental metadata & ontologies based on the Django ORM
 - **reproducible & auditable:** auto-version & timestamp execution reports, source code & compute environments, attribute records to users
 - **zero lock-in & scalable:** runs in your infrastructure; is _not_ a client for a rate-limited REST API
 - **extendable:** create custom plug-ins for your own applications based on the Django ecosystem
 - **integrations:** visualization tools like [vitessce](https://docs.lamin.ai/vitessce), workflow managers like [nextflow](https://docs.lamin.ai/nextflow) & [redun](https://docs.lamin.ai/redun), and [other tools](https://docs.lamin.ai/integrations)
 - **production-ready:** used in BigPharma, BioTech, hospitals & top labs
 
-LaminDB can be connected to LaminHub to serve as a [LIMS](https://en.wikipedia.org/wiki/Laboratory_information_management_system) for wetlab scientists, closing the drylab-wetlab feedback loop: [lamin.ai](https://lamin.ai)
+LaminDB can be connected to LaminHub to serve as a [LIMS](https://en.wikipedia.org/wiki/Laboratory_information_management_system) for wetlab scientists, closing the drylab-wetlab feedback loop: [lamin.ai](https://lamin.ai).
 
 ## Docs
 
@@ -155,26 +152,26 @@ artifact.describe()
 
 <img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/BOTCBgHDAvwglN3U0002.png" width="550">
 
-You can organize datasets with validation & annotation of any kind of metadata to then access them via queries & search. Here is a more [comprehensive example](https://lamin.ai/laminlabs/lamindata/artifact/9K1dteZ6Qx0EXK8g).
+You can organize datasets with validation & annotation of any kind of metadata to then access them via queries & search. Here is a more [comprehensive example](https://lamin.ai/laminlabs/lamindata/artifact/9K1dteZ6Qx0EXK8g):
 
 <img src="https://lamin-site-assets.s3.amazonaws.com/.lamindb/6sofuDVvTANB0f480002.png" width="850">
 
 To annotate an artifact with a label, use:
 
 ```python
-my_experiment = ln.ULabel(name="My experiment").save()  # create a label in the universal label ontology
-artifact.ulabels.add(my_experiment)  # annotate the artifact with the label
+my_experiment = ln.Record(name="My experiment").save()  # create a label in the universal label ontology
+artifact.records.add(my_experiment)  # annotate the artifact with the label
 ```
 
 To query for a set of artifacts, use the `filter()` statement.
 
 ```python
-ln.Artifact.filter(ulabels=my_experiment, suffix=".fasta").to_dataframe()  # query by suffix and the ulabel we just created
+ln.Artifact.filter(records=my_experiment, suffix=".fasta").to_dataframe()  # query by suffix and the ulabel we just created
 ln.Artifact.filter(transform__key="create-fasta.py").to_dataframe()  # query by the name of the script we just ran
 ```
 
 If you have a structured dataset like a `DataFrame`, an `AnnData`, or another array, you can validate the content of the dataset (and parse annotations).
-Here is an example for a dataframe: [docs.lamin.ai/introduction#validate-an-artifact](https://docs.lamin.ai/introduction#validate-an-artifact).
+Here is [an example for a dataframe](https://docs.lamin.ai/tutorial#validate-an-artifact).
 
 With a large body of validated datasets, you can then access data through distributed queries & batch streaming, see here: [docs.lamin.ai/arrays](https://docs.lamin.ai/arrays).