lamindb 1.11.3__py3-none-any.whl → 1.12.1__py3-none-any.whl

lamindb/models/__init__.py

@@ -54,7 +54,7 @@ from .artifact import Artifact, LazyArtifact
 from ._feature_manager import FeatureManager
 from ._label_manager import LabelManager
 from .collection import Collection, CollectionArtifact
-from .project import Person, Project, Reference
+from .project import Project, Reference
 from .query_manager import QueryManager
 from .query_set import BasicQuerySet, QuerySet, SQLRecordList
 from .artifact_set import ArtifactSet
@@ -76,9 +76,8 @@ from .project import (
     CollectionReference,
     RunProject,
     RecordProject,
-    PersonProject,
-    RecordPerson,
     RecordReference,
+    ReferenceRecord,
     ProjectRecord,
 )
 from .run import RunFeatureValue
@@ -100,7 +99,18 @@ from .record import (
     RecordArtifact,
     ArtifactRecord,
 )
-
+from .block import (
+    RootBlock,
+    ArtifactBlock,
+    TransformBlock,
+    RecordBlock,
+    CollectionBlock,
+    RunBlock,
+    SchemaBlock,
+    ProjectBlock,
+    BranchBlock,
+    SpaceBlock,
+)
 
 LinkORM = IsLink  # backward compat
 ParamValue = FeatureValue  # backward compat

lamindb/models/_django.py

@@ -98,7 +98,7 @@ def get_artifact_with_related(
 
     # Create the map that the conversion function will need.
     # It maps the target model class to the m2m field name, e.g.,
-    # {<class 'Ulabel'>: 'ulabels', <class 'CellType'>: 'cell_types'}
+    # {'Ulabel': 'ulabels', 'CellType': 'cell_types'}
     m2m_model_to_field_map = {}
     if include_m2m:
         full_map = dict_related_model_to_related_name(
@@ -109,7 +109,6 @@ def get_artifact_with_related(
             for model_cls, field_name in full_map.items()
             if not field_name.startswith("_") and field_name not in EXCLUDE_LABELS
         }
-    list(m2m_model_to_field_map.values())
     link_tables = (
         []
         if not include_feature_link

lamindb/models/_feature_manager.py

@@ -76,6 +76,7 @@ def get_accessor_by_registry_(host: Artifact | Collection) -> dict:
     }
     dictionary["Feature"] = "features"
     dictionary["ULabel"] = "ulabels"
+    dictionary["Record"] = "records"
     return dictionary
 
 

lamindb/models/_is_versioned.py

@@ -1,11 +1,11 @@
 from __future__ import annotations
 
+from pathlib import PurePosixPath
 from typing import TYPE_CHECKING, Literal, overload
 
 from django.db import models
 from lamin_utils import logger
 from lamin_utils._base62 import increment_base62
-from lamindb_setup.core.upath import LocalPathClasses, UPath
 
 from lamindb.base import ids
 from lamindb.base.fields import (
@@ -88,12 +88,20 @@ class IsVersioned(models.Model):
         """
         old_uid = self.uid  # type: ignore
         new_uid, revises = create_uid(revises=revises, version=version)
-        if self.__class__.__name__ == "Artifact" and self._key_is_virtual:
+        if (
+            self.__class__.__name__ == "Artifact"
+            and self._real_key is None
+            and (self._key_is_virtual or self.key is None)
+        ):
+            from lamindb.core.storage.paths import auto_storage_key_from_artifact_uid
+
             old_path = self.path
-            new_path = get_new_path_from_uid(
-                old_path=old_path, old_uid=old_uid, new_uid=new_uid
+            new_storage_key = auto_storage_key_from_artifact_uid(
+                new_uid, self.suffix, self._overwrite_versions
+            )
+            new_path = old_path.rename(
+                old_path.with_name(PurePosixPath(new_storage_key).name)
             )
-            new_path = UPath(old_path).rename(new_path)
             logger.success(f"updated path from {old_path} to {new_path}!")
         self.uid = new_uid
         self.version = version
@@ -193,21 +201,11 @@ def create_uid(
         if revises is not None:
             if version == revises.version:
                 raise ValueError(
-                    f"Please increment the previous version: '{revises.version}'"
+                    f"Please change the version tag or leave it `None`, '{revises.version}' is already taken"
                 )
     return suid + vuid, revises
 
 
-def get_new_path_from_uid(old_path: UPath, old_uid: str, new_uid: str):
-    if isinstance(old_path, LocalPathClasses):
-        # for local path, the rename target must be full path
-        new_path = old_path.as_posix().replace(old_uid, new_uid)
-    else:
-        # for cloud path, the rename target must be the last part of the path
-        new_path = old_path.name.replace(old_uid, new_uid)
-    return new_path
-
-
 def process_revises(
     revises: IsVersioned | None,
     version: str | None,

lamindb/models/_relations.py

@@ -83,6 +83,13 @@ def dict_related_model_to_related_name(
             record.name is not None
             and include(record.related_model)
             and record.related_model.__get_module_name__() in schema_modules
+            and not (
+                (
+                    record.related_name
+                    if not isinstance(record, ManyToManyField)
+                    else record.name
+                ).startswith("linked_in_")
+            )
         )
     }
     return d

lamindb/models/artifact.py

@@ -31,6 +31,7 @@ from lamindb.base.fields import (
     BooleanField,
     CharField,
     ForeignKey,
+    TextField,
 )
 from lamindb.errors import FieldValidationError, NoWriteAccess, UnknownStorageLocation
 from lamindb.models.query_set import QuerySet
@@ -125,6 +126,7 @@ if TYPE_CHECKING:
         ArtifactKind,
     )
     from ._label_manager import LabelManager
+    from .block import ArtifactBlock
     from .collection import Collection
     from .project import Project, Reference
     from .record import Record
@@ -419,20 +421,15 @@ def get_artifact_kwargs_from_data(
     )
 
     check_path_in_storage = False
+    real_key = None
     if use_existing_storage_key:
         inferred_key = get_relative_path_to_directory(
             path=path, directory=UPath(storage.root)
         ).as_posix()
         if key is None:
             key = inferred_key
-        else:
-            if not key == inferred_key:
-                raise InvalidArgument(
-                    f"The path '{data}' is already in registered storage"
-                    f" '{storage.root}' with key '{inferred_key}'\nYou passed"
-                    f" conflicting key '{key}': please move the file before"
-                    " registering it."
-                )
+        elif key != inferred_key:
+            real_key = inferred_key
         check_path_in_storage = True
     else:
         storage = storage
@@ -466,15 +463,16 @@ def get_artifact_kwargs_from_data(
         is_dir=n_files is not None,
     )
 
-    # do we use a virtual or an actual storage key?
-    key_is_virtual = settings.creation._artifact_use_virtual_keys
-
-    # if the file is already in storage, independent of the default
-    # we use an actual storage key
-    if check_path_in_storage:
-        key_is_virtual = False
     if overwrite_versions is None:
         overwrite_versions = n_files is not None
+
+    if check_path_in_storage:
+        # we use an actual storage key if key is not provided explicitly
+        key_is_virtual = real_key is not None
+    else:
+        # do we use a virtual or an actual storage key?
+        key_is_virtual = settings.creation._artifact_use_virtual_keys
+
     kwargs = {
         "uid": provisional_uid,
         "suffix": suffix,
@@ -493,6 +491,7 @@ def get_artifact_kwargs_from_data(
         "run": run,
         "_key_is_virtual": key_is_virtual,
         "revises": revises,
+        "_real_key": real_key,
     }
     if not isinstance(path, LocalPathClasses):
         local_filepath = None
@@ -851,6 +850,8 @@ def get_labels(
     flat_names: bool = False,
 ) -> QuerySet | dict[str, QuerySet] | list:
     """{}"""  # noqa: D415
+    from .record import Record
+
     if not isinstance(feature, Feature):
         raise TypeError("feature has to be of type Feature")
     if feature.dtype is None or not feature.dtype.startswith("cat["):
@@ -865,10 +866,14 @@ def get_labels(
     for registry in registries_to_check:
         # currently need to distinguish between ULabel and non-ULabel, because
         # we only have the feature information for Label
-        if registry == "ULabel":
+        if registry in {"ULabel", "Record"}:
             links_to_labels = get_label_links(self, registry, feature)
-            label_ids = [link.ulabel_id for link in links_to_labels]
-            qs_by_registry[registry] = ULabel.objects.using(self._state.db).filter(
+            label_ids = [
+                (link.ulabel_id if registry == "ULabel" else link.record_id)
+                for link in links_to_labels
+            ]
+            model = ULabel if registry == "ULabel" else Record
+            qs_by_registry[registry] = model.objects.using(self._state.db).filter(
                 id__in=label_ids
             )
         elif registry in self.features._accessor_by_registry:
@@ -932,7 +937,7 @@ def add_labels(
             raise ValueError(
                 "Please pass a record (a `SQLRecord` object), not a string, e.g., via:"
                 " label"
-                f" = ln.ULabel(name='{records[0]}')"  # type: ignore
+                f" = ln.Record(name='{records[0]}')"  # type: ignore
             )
         records = records_validated
 
@@ -1224,7 +1229,7 @@ class Artifact(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
             Storage locations for artifacts.
         :class:`~lamindb.Collection`
             Collections of artifacts.
-        :meth:`~lamindb.Artifact.from_df`
+        :meth:`~lamindb.Artifact.from_dataframe`
             Create an artifact from a `DataFrame`.
         :meth:`~lamindb.Artifact.from_anndata`
             Create an artifact from an `AnnData`.
@@ -1343,7 +1348,8 @@ class Artifact(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
         editable=False, unique=True, db_index=True, max_length=_len_full_uid
     )
     """A universal random id."""
-    key: str | None = CharField(db_index=True, null=True)
+    # the max length of 1024 equals the max length of a S3 key
+    key: str | None = CharField(db_index=True, null=True, max_length=1024)
     """A (virtual) relative file path within the artifact's storage location.
 
     Setting a `key` is useful to automatically group artifacts into a version family.
@@ -1353,7 +1359,10 @@ class Artifact(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
     If you register existing files in a storage location, the `key` equals the
     actual filepath on the underyling filesytem or object store.
     """
-    description: str | None = CharField(db_index=True, null=True)
+    _real_key: str | None = CharField(db_index=True, null=True, max_length=1024)
+    """An optional real storage key."""
+    # db_index on description because sometimes we query for equality in the case of artifacts
+    description: str | None = TextField(null=True, db_index=True)
     """A description."""
     storage: Storage = ForeignKey(
         Storage, PROTECT, related_name="artifacts", editable=False
@@ -1408,10 +1417,6 @@ class Artifact(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
         max_length=30, db_index=True, null=True, editable=False
     )
     """Type of hash."""
-    ulabels: ULabel = models.ManyToManyField(
-        ULabel, through="ArtifactULabel", related_name="artifacts"
-    )
-    """The ulabels measured in the artifact (:class:`~lamindb.ULabel`)."""
     run: Run | None = ForeignKey(
         Run,
         PROTECT,
@@ -1463,15 +1468,28 @@ class Artifact(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
     )
     """Creator of record."""
     _overwrite_versions: bool = BooleanField(default=None)
-    # see corresponding property `overwrite_versions`
+    """See corresponding property `overwrite_versions`."""
+    ulabels: ULabel = models.ManyToManyField(
+        ULabel, through="ArtifactULabel", related_name="artifacts"
+    )
+    """The ulabels annotating this artifact."""
+    users: User = models.ManyToManyField(
+        User,
+        through="ArtifactUser",
+        through_fields=("artifact", "user"),
+        related_name="+",
+    )
+    """The users annotating this artifact."""
     projects: Project
-    """Annotating projects."""
+    """The projects annotating this artifact."""
     references: Reference
-    """Annotating references."""
+    """The references annotating this artifact."""
     records: Record
-    """Annotating records."""
+    """The records annotating this artifact."""
     linked_in_records: Record
-    """Linked in records."""
+    """This artifact is linked in these records as a value."""
+    blocks: ArtifactBlock
+    """The blocks that annotate this artifact."""
 
     @overload
     def __init__(
@@ -2459,34 +2477,42 @@ class Artifact(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
             err_msg += " with " + ("a folder." if _overwrite_versions else "a file.")
             raise ValueError(err_msg)
 
-        if self.key is not None and not self._key_is_virtual:
-            key_path = PurePosixPath(self.key)
-            new_filename = f"{key_path.stem}{kwargs['suffix']}"
-            # the following will only be true if the suffix changes!
-            if key_path.name != new_filename:
-                self._clear_storagekey = self.key
-                self.key = str(key_path.with_name(new_filename))
+        new_suffix = kwargs["suffix"]
+        if new_suffix != self.suffix:
+            key = self.key
+            real_key = self._real_key
+            if key is not None:
+                new_key = PurePosixPath(key).with_suffix(new_suffix).as_posix()
+            else:
+                new_key = None
+            if (key is not None and not self._key_is_virtual) or real_key is not None:
+                # real_key is not None implies key is not None
+                assert key is not None  # noqa: S101
+                if real_key is not None:
+                    self._clear_storagekey = real_key
+                    self._real_key = (
+                        PurePosixPath(real_key).with_suffix(new_suffix).as_posix()
+                    )
+                    warn_msg = f", _real_key '{real_key}' with '{self._real_key}'"
+                else:
+                    self._clear_storagekey = key
+                    warn_msg = ""
+                self.key = new_key
                 # update old key with the new one so that checks in record pass
-                self._old_key = self.key
+                self._old_key = new_key
                 logger.warning(
-                    f"replacing the file will replace key '{key_path}' with '{self.key}'"
-                    f" and delete '{key_path}' upon `save()`"
+                    f"replacing the file will replace key '{key}' with '{new_key}'{warn_msg}"
+                    f" and delete '{self._clear_storagekey}' upon `save()`"
                 )
-        else:
-            old_storage = auto_storage_key_from_artifact(self)
-            is_dir = self.n_files is not None
-            new_storage = auto_storage_key_from_artifact_uid(
-                self.uid, kwargs["suffix"], is_dir
-            )
-            if old_storage != new_storage:
-                self._clear_storagekey = old_storage
-                if self.key is not None:
-                    new_key_path = PurePosixPath(self.key).with_suffix(kwargs["suffix"])
-                    self.key = str(new_key_path)
-                    # update old key with the new one so that checks in record pass
-                    self._old_key = self.key
-
-        self.suffix = kwargs["suffix"]
+            else:
+                # purely virtual key case
+                self._clear_storagekey = auto_storage_key_from_artifact(self)
+                # might replace None with None, not a big deal
+                self.key = new_key
+                # update the old key with the new one so that checks in record pass
+                self._old_key = new_key
+
+        self.suffix = new_suffix
         self.size = kwargs["size"]
         self.hash = kwargs["hash"]
         self._hash_type = kwargs["_hash_type"]
@@ -3025,6 +3051,23 @@ class ArtifactFeatureValue(BaseSQLRecord, IsLink, TracksRun):
         unique_together = ("artifact", "featurevalue")
 
 
+class ArtifactUser(BaseSQLRecord, IsLink, TracksRun):
+    id: int = models.BigAutoField(primary_key=True)
+    artifact: Artifact = ForeignKey("Artifact", CASCADE, related_name="links_user")
+    user: User = ForeignKey(User, PROTECT, related_name="links_artifact")
+    feature: Feature | None = ForeignKey(
+        Feature, PROTECT, null=True, related_name="links_artifactuser", default=None
+    )
+    label_ref_is_name: bool | None = BooleanField(null=True)
+    feature_ref_is_name: bool | None = BooleanField(null=True)
+
+    class Meta:
+        # can have the same label linked to the same artifact if the feature is
+        # different
+        app_label = "lamindb"
+        unique_together = ("artifact", "user", "feature")
+
+
 def _track_run_input(
     data: (
         Artifact | Iterable[Artifact]

lamindb/models/artifact_set.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from collections.abc import Iterable, Iterator
 from typing import TYPE_CHECKING, Literal
 
-from django.db.models import Q, TextField, Value
+from django.db.models import Case, Q, TextField, Value, When
 from django.db.models.functions import Concat
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
@@ -143,11 +143,13 @@ def artifacts_from_path(artifacts: ArtifactSet, path: UPathStr) -> ArtifactSet:
     if stem_len == 16:
         qs = artifacts.filter(
             Q(_key_is_virtual=True) | Q(key__isnull=True),
+            _real_key__isnull=True,
             uid__startswith=stem,
         )
     elif stem_len == 20:
         qs = artifacts.filter(
             Q(_key_is_virtual=True) | Q(key__isnull=True),
+            _real_key__isnull=True,
             uid=stem,
         )
     else:
@@ -157,10 +159,25 @@ def artifacts_from_path(artifacts: ArtifactSet, path: UPathStr) -> ArtifactSet:
         return qs
 
     qs = (
-        artifacts.filter(_key_is_virtual=False)
+        artifacts.filter(Q(_key_is_virtual=False) | Q(_real_key__isnull=False))
         .alias(
-            db_path=Concat("storage__root", Value("/"), "key", output_field=TextField())
+            db_path=Case(
+                When(
+                    _real_key__isnull=False,
+                    then=Concat(
+                        "storage__root",
+                        Value("/"),
+                        "_real_key",
+                        output_field=TextField(),
+                    ),
+                ),
+                default=Concat(
+                    "storage__root", Value("/"), "key", output_field=TextField()
+                ),
+                output_field=TextField(),
+            )
         )
         .filter(db_path=path_str)
     )
+
     return qs

lamindb/models/block.py

@@ -0,0 +1,174 @@
+from datetime import datetime
+
+from django.db import models
+from django.db.models import CASCADE, CharField, DateTimeField, ForeignKey, TextField
+
+from .artifact import Artifact
+from .collection import Collection
+from .feature import Feature
+from .project import Project
+from .record import Record
+from .run import Run, User
+from .schema import Schema
+from .sqlrecord import BaseSQLRecord, Branch, IsVersioned, Space
+from .transform import Transform
+
+
+class BlockMixin(IsVersioned):
+    class Meta:
+        abstract = True
+
+    _len_full_uid: int = 20
+    _len_stem_uid: int = 16
+
+    id = models.BigAutoField(primary_key=True)
+    """Internal id, valid only in one DB instance."""
+    uid: str = CharField(
+        editable=False, unique=True, db_index=True, max_length=_len_full_uid
+    )
+    """Universal id."""
+    content: str = TextField()
+    """Content of the block."""
+    hash: str = CharField(max_length=22, db_index=True, null=True)
+    """Content hash of the block."""
+    type: str = CharField(
+        max_length=22, db_index=True, default="mdpage", db_default="mdpage"
+    )
+    """Type of the block.
+
+    Only current option is: "mdpage" (markdown page).
+    """
+    vertical_pos: float = models.FloatField(default=0, db_default=0, db_index=True)
+    """The vertical position of the block in the GUI as a float.
+
+    It can be used to act as a percentage to define absolute positioning or for ordering.
+    """
+    created_at: datetime = DateTimeField(
+        editable=False, db_default=models.functions.Now(), db_index=True
+    )
+    """Time of creation of record."""
+    created_by: User = ForeignKey(
+        "User", CASCADE, default=None, related_name="+", null=True
+    )
+    """Creator of block."""
+
+
+class RootBlock(BlockMixin, BaseSQLRecord):
+    """A root block for every registry that can appear at the top of the registry root block in the GUI."""
+
+    class Meta:
+        app_label = "lamindb"
+
+    name: str = CharField(max_length=255, db_index=True)
+    """The entity for which we want to create a block.
+
+    Conventions are mostly to take the SQL table name:
+        name = "instance" means instance
+        name = "lamindb_artifact" means artifact
+        name = "lamindb_transform" means transform
+        name = "bionty_celltype" means bionty cell type
+    """
+
+
+class RecordBlock(BlockMixin, BaseSQLRecord):
+    """An unstructured notes block that can be attached to an artifact."""
+
+    class Meta:
+        app_label = "lamindb"
+
+    record: Record = ForeignKey(Record, CASCADE, related_name="blocks")
+    """The record to which the block is attached."""
+
+
+class ArtifactBlock(BlockMixin, BaseSQLRecord):
+    """An unstructured notes block that can be attached to an artifact."""
+
+    class Meta:
+        app_label = "lamindb"
+
+    artifact: Artifact = ForeignKey(Artifact, CASCADE, related_name="blocks")
+    """The artifact to which the block is attached."""
+
+
+class TransformBlock(BlockMixin, BaseSQLRecord):
+    """An unstructured notes block that can be attached to an artifact."""
+
+    class Meta:
+        app_label = "lamindb"
+
+    transform: Transform = ForeignKey(
+        Transform, CASCADE, related_name="blocks", null=True
+    )
+    """The transform to which the block is attached."""
+    line_number: int | None = models.IntegerField(null=True)
+    """The line number in the source code to which the block belongs."""
+
+
+class RunBlock(BlockMixin, BaseSQLRecord):
+    """An unstructured notes block that can be attached to an artifact."""
+
+    class Meta:
+        app_label = "lamindb"
+
+    run: Run = ForeignKey(Run, CASCADE, related_name="blocks")
+    """The run to which the block is attached."""
+
+
+class CollectionBlock(BlockMixin, BaseSQLRecord):
+    """An unstructured notes block that can be attached to an artifact."""
+
+    class Meta:
+        app_label = "lamindb"
+
+    collection: Collection = ForeignKey(
+        Collection, CASCADE, related_name="blocks", null=True
+    )
+    """The collection to which the block is attached."""
+
+
+class SchemaBlock(BlockMixin, BaseSQLRecord):
+    """An unstructured notes block that can be attached to an artifact."""
+
+    class Meta:
+        app_label = "lamindb"
+
+    schema: Schema = ForeignKey(Schema, CASCADE, related_name="blocks")
+    """The schema to which the block is attached."""
+
+
+class FeatureBlock(BlockMixin, BaseSQLRecord):
+    """An unstructured notes block that can be attached to an artifact."""
+
+    class Meta:
+        app_label = "lamindb"
+
+    feature: Feature = ForeignKey(Feature, CASCADE, related_name="blocks")
+    """The feature to which the block is attached."""
+
+
+class ProjectBlock(BlockMixin, BaseSQLRecord):
+    """An unstructured notes block that can be attached to an artifact."""
+
+    class Meta:
+        app_label = "lamindb"
+
+    project: Project = ForeignKey(Project, CASCADE, related_name="blocks")
+
+
+class SpaceBlock(BlockMixin, BaseSQLRecord):
+    """An unstructured notes block that can be attached to an artifact."""
+
+    class Meta:
+        app_label = "lamindb"
+
+    space: Space = ForeignKey(Space, CASCADE, related_name="blocks")
+    """The space to which the block is attached."""
+
+
+class BranchBlock(BlockMixin, BaseSQLRecord):
+    """An unstructured notes block that can be attached to an artifact."""
+
+    class Meta:
+        app_label = "lamindb"
+
+    branch: Branch = ForeignKey(Branch, CASCADE, related_name="blocks")