lamindb 1.10.1__py3-none-any.whl → 1.11.0__py3-none-any.whl

lamindb/models/artifact_set.py

@@ -3,8 +3,11 @@ from __future__ import annotations
 from collections.abc import Iterable, Iterator
 from typing import TYPE_CHECKING, Literal
 
+from django.db.models import Q, TextField, Value
+from django.db.models.functions import Concat
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
+from upath import UPath
 
 from ..core._mapped_collection import MappedCollection
 from ..core.storage._backed_access import _open_dataframe
@@ -13,10 +16,10 @@ from .collection import Collection, _load_concat_artifacts
 
 if TYPE_CHECKING:
     from anndata import AnnData
+    from lamindb_setup.types import UPathStr
     from pandas import DataFrame
     from polars import LazyFrame as PolarsLazyFrame
     from pyarrow.dataset import Dataset as PyArrowDataset
-    from upath import UPath
 
 
 UNORDERED_WARNING = (
@@ -25,6 +28,7 @@ UNORDERED_WARNING = (
 )
 
 
+# maybe make this abstract
 class ArtifactSet(Iterable):
     """Abstract class representing sets of artifacts returned by queries.
 
@@ -120,3 +124,43 @@ class ArtifactSet(Iterable):
         # track only if successful
         _track_run_input(artifacts, is_run_input)
         return ds
+
+
+def artifacts_from_path(artifacts: ArtifactSet, path: UPathStr) -> ArtifactSet:
+    """Returns artifacts in the query set that are registered for the provided path."""
+    from lamindb.models import BasicQuerySet, QuerySet
+
+    # not QuerySet but only BasicQuerySet
+    assert isinstance(artifacts, BasicQuerySet) and not isinstance(artifacts, QuerySet)  # noqa: S101
+
+    upath = UPath(path)
+
+    path_str = upath.as_posix()
+
+    stem = upath.stem
+    stem_len = len(stem)
+
+    if stem_len == 16:
+        qs = artifacts.filter(
+            Q(_key_is_virtual=True) | Q(key__isnull=True),
+            uid__startswith=stem,
+        )
+    elif stem_len == 20:
+        qs = artifacts.filter(
+            Q(_key_is_virtual=True) | Q(key__isnull=True),
+            uid=stem,
+        )
+    else:
+        qs = None
+
+    if qs:  # an empty query set evaluates to False
+        return qs
+
+    qs = (
+        artifacts.filter(_key_is_virtual=False)
+        .alias(
+            db_path=Concat("storage__root", Value("/"), "key", output_field=TextField())
+        )
+        .filter(db_path=path_str)
+    )
+    return qs

lamindb/models/can_curate.py

@@ -580,8 +580,7 @@ class CanCurate:
         """Bulk create validated records by parsing values for an identifier such as a name or an id).
 
         Args:
-            values: A list of values for an identifier, e.g.
-                `["name1", "name2"]`.
+            values: A list of values for an identifier, e.g. `["name1", "name2"]`.
             field: A `SQLRecord` field to look up, e.g., `bt.CellMarker.name`.
             create: Whether to create records if they don't exist.
             organism: A `bionty.Organism` name or record.

lamindb/models/collection.py

@@ -153,6 +153,7 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
 
     class Meta(SQLRecord.Meta, IsVersioned.Meta, TracksRun.Meta, TracksUpdates.Meta):
         abstract = False
+        app_label = "lamindb"
 
     _len_full_uid: int = 20
     _len_stem_uid: int = 16
@@ -400,7 +401,7 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
 
         """
         return Collection(  # type: ignore
-            self.artifacts.all().list() + [artifact],
+            self.artifacts.all().to_list() + [artifact],
             # key is automatically derived from revises.key
             description=self.description,
             revises=self,
@@ -576,39 +577,6 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
         _track_run_input(self, is_run_input)
         return concat_object
 
-    def delete(self, permanent: bool | None = None) -> None:
-        """Delete collection.
-
-        Args:
-            permanent: Whether to permanently delete the collection record (skips trash).
-
-        Examples:
-
-            For any `Collection` object `collection`, call:
-
-            >>> collection.delete()
-        """
-        # change branch_id to trash
-        trash_branch_id = -1
-        if self.branch_id > trash_branch_id and permanent is not True:
-            self.branch_id = trash_branch_id
-            self.save()
-            logger.warning(f"moved collection to trash (branch_id = {trash_branch_id})")
-            return
-
-        # permanent delete
-        if permanent is None:
-            response = input(
-                "Collection record is already in trash! Are you sure to delete it from your"
-                " database? (y/n) You can't undo this action."
-            )
-            delete_record = response == "y"
-        else:
-            delete_record = permanent
-
-        if delete_record:
-            super().delete()
-
     def save(self, using: str | None = None) -> Collection:
         """Save the collection and underlying artifacts to database & storage.
 
@@ -728,6 +696,7 @@ class CollectionArtifact(BaseSQLRecord, IsLink, TracksRun):
     artifact: Artifact = ForeignKey(Artifact, PROTECT, related_name="links_collection")
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("collection", "artifact")
 
 

lamindb/models/feature.py

@@ -11,6 +11,7 @@ from django.db.models.query_utils import DeferredAttribute
 from django.db.utils import IntegrityError
 from lamin_utils import logger
 from lamindb_setup._init_instance import get_schema_module_name
+from lamindb_setup.core import deprecated
 from lamindb_setup.core.hashing import HASH_LENGTH, hash_dict, hash_string
 from lamindb_setup.errors import (
     MODULE_WASNT_CONFIGURED_MESSAGE_TEMPLATE,
@@ -598,6 +599,7 @@ class Feature(SQLRecord, CanCurate, TracksRun, TracksUpdates):
 
     class Meta(SQLRecord.Meta, TracksRun.Meta, TracksUpdates.Meta):
         abstract = False
+        app_label = "lamindb"
 
     _name_field: str = "name"
     _aux_fields: dict[str, tuple[str, type]] = {
@@ -776,12 +778,21 @@ class Feature(SQLRecord, CanCurate, TracksRun, TracksUpdates):
                 )
 
     @classmethod
-    def from_df(cls, df: pd.DataFrame, field: FieldAttr | None = None) -> SQLRecordList:
-        """Create Feature records for columns."""
+    def from_dataframe(
+        cls, df: pd.DataFrame, field: FieldAttr | None = None, *, mute: bool = False
+    ) -> SQLRecordList:
+        """Create Feature records for dataframe columns.
+
+        Args:
+            df: Source DataFrame to extract column information from
+            field: FieldAttr for Feature model validation, defaults to Feature.name
+            mute: Whether to mute Feature creation similar names found warnings
+        """
         field = Feature.name if field is None else field
         registry = field.field.model  # type: ignore
         if registry != Feature:
             raise ValueError("field must be a Feature FieldAttr!")
+
         categoricals = categoricals_from_df(df)
         dtypes = {}
         for name, col in df.items():
@@ -789,15 +800,107 @@ class Feature(SQLRecord, CanCurate, TracksRun, TracksUpdates):
                 dtypes[name] = "cat"
             else:
                 dtypes[name] = serialize_pandas_dtype(col.dtype)
-        with logger.mute():  # silence the warning "loaded record with exact same name "
+
+        if mute:
+            original_verbosity = logger._verbosity
+            logger.set_verbosity(0)
+        try:
             features = [
                 Feature(name=name, dtype=dtype) for name, dtype in dtypes.items()
             ]  # type: ignore
-        assert len(features) == len(df.columns)  # noqa: S101
-        return SQLRecordList(features)
+            assert len(features) == len(df.columns)  # noqa: S101
+            return SQLRecordList(features)
+        finally:
+            if mute:
+                logger.set_verbosity(original_verbosity)
+
+    @classmethod
+    @deprecated("from_dataframe")
+    def from_df(
+        cls, df: pd.DataFrame, field: FieldAttr | None = None, *, mute: bool = False
+    ) -> SQLRecordList:
+        return cls.from_dataframe(df, field, mute=mute)
+
+    @classmethod
+    def from_dict(
+        cls,
+        dictionary: dict[str, Any],
+        field: FieldAttr | None = None,
+        *,
+        str_as_cat: bool | None = None,
+        mute: bool = False,
+    ) -> SQLRecordList:
+        """Create Feature records for dictionary keys.
+
+        Args:
+            dictionary: Source dictionary to extract key information from
+            field: FieldAttr for Feature model validation, defaults to Feature.name
+            str_as_cat: Whether to interpret string values as categorical
+            mute: Whether to mute dtype inference and feature creation warnings
+        """
+        from lamindb.models._feature_manager import infer_feature_type_convert_json
+
+        field = Feature.name if field is None else field
+        registry = field.field.model  # type: ignore
+        if registry != Feature:
+            raise ValueError("field must be a Feature FieldAttr!")
+
+        dtypes = {}
+        ambiguous_keys = []
+        for key, value in dictionary.items():
+            dtype, _, message = infer_feature_type_convert_json(key, value, mute=mute)
+
+            if dtype == "cat ? str":
+                if str_as_cat is None:
+                    ambiguous_keys.append(
+                        (key, "str or cat", message.strip("# ") if message else "")
+                    )
+                    continue
+                if str_as_cat:
+                    dtype = "cat"
+                else:
+                    dtype = "str"
+
+            elif dtype == "list[cat ? str]":
+                if str_as_cat is None:
+                    ambiguous_keys.append(
+                        (
+                            key,
+                            "list[str] or list[cat]",
+                            message.strip("# ") if message else "",
+                        )
+                    )
+                    continue
+                if str_as_cat:
+                    dtype = "list[cat]"
+                else:
+                    dtype = "list[str]"
+
+            dtypes[key] = dtype
+
+        if ambiguous_keys:
+            error_msg = "Ambiguous dtypes detected. Please pass `str_as_cat` parameter or create features explicitly:\n"
+            for key, options, msg in ambiguous_keys:
+                error_msg += f"  '{key}': {options}"
+                if msg:
+                    error_msg += f" ({msg})"
+                error_msg += "\n"
+            error_msg += "\nUse `str_as_cat=True` to treat strings as categorical, or `str_as_cat=False` for plain strings."
+            raise ValueError(error_msg)
+
+        if mute:
+            original_verbosity = logger._verbosity
+            logger.set_verbosity(0)
+        try:
+            features = [Feature(name=key, dtype=dtype) for key, dtype in dtypes.items()]  # type: ignore
+            assert len(features) == len(dictionary)  # noqa: S101
+            return SQLRecordList(features)
+        finally:
+            if mute:
+                logger.set_verbosity(original_verbosity)
 
     def save(self, *args, **kwargs) -> Feature:
-        """Save."""
+        """Save the feature to the instance."""
         super().save(*args, **kwargs)
         return self
 
@@ -886,7 +989,7 @@ class Feature(SQLRecord, CanCurate, TracksRun, TracksUpdates):
     #     However, when accessing an artifact annotation with a feature that's defined on the observation-level, say `"cell_type"`, you expect a set of values. So,
     #     `artifact.features.get_values(["cell_type_from_expert"])` should return a set: `{"T cell", "B cell"}`.
 
-    #     The value of `observational_unit` is currently auto-managed: if using `artifact.featueres.add_values()`,
+    #     The value of `observational_unit` is currently auto-managed: if using `artifact.features.add_values()`,
     #     it will be set to `Artifact`. In a curator, the value depends on whether it's an artifact- or observation-level slot
     #     (e.g. `.uns` is artifact-level in `AnnData` whereas `.obs` is observation-level).
 
@@ -927,6 +1030,7 @@ class FeatureValue(SQLRecord, TracksRun):
     """Value hash."""
 
     class Meta(BaseSQLRecord.Meta, TracksRun.Meta):
+        app_label = "lamindb"
         unique_together = ("feature", "hash")
 
     @classmethod

lamindb/models/has_parents.py

@@ -388,7 +388,7 @@ def _df_edges_from_parents(
     )
     all = record.__class__.objects
     records = parents | all.filter(id=record.id)
-    df = records.distinct().df(include=[f"{key}__id"])
+    df = records.distinct().to_dataframe(include=[f"{key}__id"])
     if f"{key}__id" not in df.columns:
         return None
     df_edges = df[[f"{key}__id"]]
@@ -494,21 +494,21 @@ def _get_all_parent_runs(data: Artifact | Collection) -> list:
                 r.__getattribute__(f"input_{name}s")
                 .all()
                 .filter(branch_id__in=[0, 1])
-                .list()
+                .to_list()
             )
             if name == "artifact":
                 inputs_run += (
-                    r.input_collections.all().filter(branch_id__in=[0, 1]).list()
+                    r.input_collections.all().filter(branch_id__in=[0, 1]).to_list()
                 )
             outputs_run = (
                 r.__getattribute__(f"output_{name}s")
                 .all()
                 .filter(branch_id__in=[0, 1])
-                .list()
+                .to_list()
             )
             if name == "artifact":
                 outputs_run += (
-                    r.output_collections.all().filter(branch_id__in=[0, 1]).list()
+                    r.output_collections.all().filter(branch_id__in=[0, 1]).to_list()
                 )
             # if inputs are outputs artifacts are the same, will result infinite loop
             # so only show as outputs
@@ -554,11 +554,11 @@ def _get_all_child_runs(data: Artifact | Collection) -> list:
                 r.__getattribute__(f"input_{name}s")
                 .all()
                 .filter(branch_id__in=[0, 1])
-                .list()
+                .to_list()
             )
             if name == "artifact":
                 inputs_run += (
-                    r.input_collections.all().filter(branch_id__in=[0, 1]).list()
+                    r.input_collections.all().filter(branch_id__in=[0, 1]).to_list()
                 )
             run_inputs_outputs += [(inputs_run, r)]
 
@@ -566,25 +566,25 @@ def _get_all_child_runs(data: Artifact | Collection) -> list:
                 r.__getattribute__(f"output_{name}s")
                 .all()
                 .filter(branch_id__in=[0, 1])
-                .list()
+                .to_list()
             )
             if name == "artifact":
                 outputs_run += (
-                    r.output_collections.all().filter(branch_id__in=[0, 1]).list()
+                    r.output_collections.all().filter(branch_id__in=[0, 1]).to_list()
                 )
             run_inputs_outputs += [(r, outputs_run)]
 
             child_runs.update(
                 Run.filter(  # type: ignore
                     **{f"input_{name}s__uid__in": [i.uid for i in outputs_run]}
-                ).list()
+                ).to_list()
             )
             # for artifacts, also include collections in the lineage
             if name == "artifact":
                 child_runs.update(
                     Run.filter(  # type: ignore
                         input_collections__uid__in=[i.uid for i in outputs_run]
-                    ).list()
+                    ).to_list()
                 )
         runs = child_runs
     return run_inputs_outputs

lamindb/models/project.py

@@ -53,6 +53,7 @@ class Person(SQLRecord, CanCurate, TracksRun, TracksUpdates, ValidateFields):
 
     class Meta(SQLRecord.Meta, TracksRun.Meta, TracksUpdates.Meta):
         abstract = False
+        app_label = "lamindb"
 
     id: int = models.AutoField(primary_key=True)
     """Internal id, valid only in one DB instance."""
@@ -107,6 +108,7 @@ class Reference(SQLRecord, CanCurate, TracksRun, TracksUpdates, ValidateFields):
 
     class Meta(SQLRecord.Meta, TracksRun.Meta, TracksUpdates.Meta):
         abstract = False
+        app_label = "lamindb"
 
     id: int = models.AutoField(primary_key=True)
     """Internal id, valid only in one DB instance."""
@@ -215,6 +217,7 @@ class Project(SQLRecord, CanCurate, TracksRun, TracksUpdates, ValidateFields):
 
     class Meta(SQLRecord.Meta, TracksRun.Meta, TracksUpdates.Meta):
         abstract = False
+        app_label = "lamindb"
 
     id: int = models.AutoField(primary_key=True)
     """Internal id, valid only in one DB instance."""
@@ -286,10 +289,14 @@ class Project(SQLRecord, CanCurate, TracksRun, TracksUpdates, ValidateFields):
         Schema, through="SchemaProject", related_name="projects"
     )
     """Linked schemas."""
-    records: Record = models.ManyToManyField(
+    linked_in_records: Record = models.ManyToManyField(
         Record, through="RecordProject", related_name="linked_projects"
     )
     """Linked records."""
+    records: Record = models.ManyToManyField(
+        Record, through="ProjectRecord", related_name="projects"
+    )
+    """Annotated record."""
     collections: Collection = models.ManyToManyField(
         Collection, through="CollectionProject", related_name="projects"
     )
@@ -336,6 +343,7 @@ class ArtifactProject(BaseSQLRecord, IsLink, TracksRun):
     feature_ref_is_name: bool | None = BooleanField(null=True, default=None)
 
     class Meta:
+        app_label = "lamindb"
         # can have the same label linked to the same artifact if the feature is different
         unique_together = ("artifact", "project", "feature")
 
@@ -358,6 +366,7 @@ class RunProject(BaseSQLRecord, IsLink):
     """Creator of record."""
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("run", "project")
 
 
@@ -367,6 +376,7 @@ class TransformProject(BaseSQLRecord, IsLink, TracksRun):
     project: Project = ForeignKey(Project, PROTECT, related_name="links_transform")
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("transform", "project")
 
 
@@ -378,6 +388,7 @@ class CollectionProject(BaseSQLRecord, IsLink, TracksRun):
     project: Project = ForeignKey(Project, PROTECT, related_name="links_collection")
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("collection", "project")
 
 
@@ -387,6 +398,7 @@ class ULabelProject(BaseSQLRecord, IsLink, TracksRun):
     project: Project = ForeignKey(Project, PROTECT, related_name="links_ulabel")
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("ulabel", "project")
 
 
@@ -397,6 +409,7 @@ class PersonProject(BaseSQLRecord, IsLink, TracksRun):
     role: str | None = CharField(null=True, default=None)
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("person", "project")
 
 
@@ -406,6 +419,7 @@ class FeatureProject(BaseSQLRecord, IsLink, TracksRun):
     project: Project = ForeignKey(Project, PROTECT, related_name="links_feature")
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("feature", "project")
 
 
@@ -415,6 +429,7 @@ class SchemaProject(BaseSQLRecord, IsLink, TracksRun):
     project: Project = ForeignKey(Project, PROTECT, related_name="links_schema")
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("schema", "project")
 
 
@@ -425,6 +440,7 @@ class RecordPerson(BaseSQLRecord, IsLink):
     value: Person = ForeignKey(Person, PROTECT, related_name="links_record")
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("record", "feature", "value")
 
 
@@ -437,16 +453,37 @@ class RecordReference(BaseSQLRecord, IsLink):
     value: Reference = ForeignKey(Reference, PROTECT, related_name="links_record")
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("record", "feature", "value")
 
 
+# for annotation of records with projects, RecordProject is for storing project values
+class ProjectRecord(BaseSQLRecord, IsLink, TracksRun):
+    id: int = models.BigAutoField(primary_key=True)
+    record: Record = ForeignKey(Record, CASCADE, related_name="links_project")
+    project: Project = ForeignKey(Project, PROTECT, related_name="links_record")
+    feature: Feature | None = ForeignKey(
+        Feature,
+        PROTECT,
+        null=True,
+        default=None,
+        related_name="links_projectrecord",
+    )
+
+    class Meta:
+        # can have the same label linked to the same artifact if the feature is different
+        app_label = "lamindb"
+        unique_together = ("record", "project", "feature")
+
+
 class RecordProject(BaseSQLRecord, IsLink):
     id: int = models.BigAutoField(primary_key=True)
     record: Record = ForeignKey(Record, CASCADE, related_name="values_project")
     feature: Feature = ForeignKey(Feature, PROTECT, related_name="links_recordproject")
-    value: Project = ForeignKey(Project, PROTECT, related_name="links_record")
+    value: Project = ForeignKey(Project, PROTECT, related_name="links_in_record")
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("record", "feature", "value")
 
 
@@ -465,6 +502,7 @@ class ArtifactReference(BaseSQLRecord, IsLink, TracksRun):
     feature_ref_is_name: bool | None = BooleanField(null=True, default=None)
 
     class Meta:
+        app_label = "lamindb"
         # can have the same label linked to the same artifact if the feature is different
         unique_together = ("artifact", "reference", "feature")
 
@@ -479,6 +517,7 @@ class TransformReference(BaseSQLRecord, IsLink, TracksRun):
     )
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("transform", "reference")
 
 
@@ -492,4 +531,5 @@ class CollectionReference(BaseSQLRecord, IsLink, TracksRun):
     )
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("collection", "reference")

lamindb/models/query_manager.py

@@ -25,6 +25,7 @@ from django.db.models.lookups import (
 )
 from lamin_utils import logger
 from lamin_utils._lookup import Lookup
+from lamindb_setup.core import deprecated
 from lamindb_setup.core._docs import doc_args
 
 if TYPE_CHECKING:
@@ -241,7 +242,7 @@ class QueryManager(Manager):
         >>> label = ln.ULabel.get(name="ULabel1")
         >>> label.parents.set(labels)
         >>> manager = label.parents
-        >>> manager.df()
+        >>> manager.to_dataframe()
     """
 
     def _track_run_input_manager(self):
@@ -264,7 +265,7 @@ class QueryManager(Manager):
                     logger.warning(WARNING_RUN_TRANSFORM)
                 _track_run_input(self.instance)
 
-    def list(self, field: str | None = None):
+    def to_list(self, field: str | None = None):
         """Populate a list with the results.
 
         Examples:
@@ -273,8 +274,8 @@ class QueryManager(Manager):
             >>> ln.ULabel(name="ULabel1").save()
             >>> label = ln.ULabel.get(name="ULabel1")
             >>> label.parents.set(labels)
-            >>> label.parents.list()
-            >>> label.parents.list("name")
+            >>> label.parents.to_list()
+            >>> label.parents.to_list("name")
             ['ULabel1', 'ULabel2', 'ULabel3']
         """
         if field is None:
@@ -283,12 +284,20 @@ class QueryManager(Manager):
             self._track_run_input_manager()
             return list(self.values_list(field, flat=True))
 
-    def df(self, **kwargs):
+    @deprecated(new_name="to_list")
+    def list(self, field: str | None = None):
+        return self.to_list(field)
+
+    def to_dataframe(self, **kwargs):
         """Convert to DataFrame.
 
-        For `**kwargs`, see :meth:`lamindb.models.QuerySet.df`.
+        For `**kwargs`, see :meth:`lamindb.models.QuerySet.to_dataframe`.
         """
-        return self.all().df(**kwargs)
+        return self.all().to_dataframe(**kwargs)
+
+    @deprecated(new_name="to_dataframe")
+    def df(self, **kwargs):
+        return self.to_dataframe(**kwargs)
 
     def all(self):
         """Return QuerySet of all.