lamindb 1.10.2__py3-none-any.whl → 1.11a1__py3-none-any.whl

lamindb/models/query_set.py

@@ -13,6 +13,7 @@ from django.db import models
 from django.db.models import F, ForeignKey, ManyToManyField, Q, Subquery
 from django.db.models.fields.related import ForeignObjectRel
 from lamin_utils import logger
+from lamindb_setup.core import deprecated
 from lamindb_setup.core._docs import doc_args
 
 from ..errors import DoesNotExist
@@ -144,7 +145,6 @@ def process_expressions(queryset: QuerySet, expressions: dict) -> dict:
         queryset,
         expressions,
     )
-
     if issubclass(queryset.model, SQLRecord):
         # branch_id is set to 1 unless expressions contains id or uid
         if not (
@@ -173,32 +173,28 @@ def process_expressions(queryset: QuerySet, expressions: dict) -> dict:
 
 
 def get(
-    registry_or_queryset: Union[type[SQLRecord], QuerySet],
+    registry_or_queryset: Union[type[SQLRecord], BasicQuerySet],
     idlike: int | str | None = None,
     **expressions,
 ) -> SQLRecord:
-    if isinstance(registry_or_queryset, QuerySet):
+    if isinstance(registry_or_queryset, BasicQuerySet):
         qs = registry_or_queryset
         registry = qs.model
     else:
-        qs = QuerySet(model=registry_or_queryset)
+        qs = BasicQuerySet(model=registry_or_queryset)
         registry = registry_or_queryset
     if isinstance(idlike, int):
-        return super(QuerySet, qs).get(id=idlike)  # type: ignore
+        return BasicQuerySet.get(qs, id=idlike)
     elif isinstance(idlike, str):
-        qs = qs.filter(uid__startswith=idlike)
-
         NAME_FIELD = (
             registry._name_field if hasattr(registry, "_name_field") else "name"
         )
         DOESNOTEXIST_MSG = f"No record found with uid '{idlike}'. Did you forget a keyword as in {registry.__name__}.get({NAME_FIELD}='{idlike}')?"
-
-        if issubclass(registry, IsVersioned):
-            if len(idlike) <= registry._len_stem_uid:
-                return one_helper(qs.latest_version(), DOESNOTEXIST_MSG)
-            else:
-                return one_helper(qs, DOESNOTEXIST_MSG)
+        if issubclass(registry, IsVersioned) and len(idlike) <= registry._len_stem_uid:
+            qs = BasicQuerySet.filter(qs, uid__startswith=idlike, is_latest=True)
+            return one_helper(qs, DOESNOTEXIST_MSG)
         else:
+            qs = BasicQuerySet.filter(qs, uid__startswith=idlike)
             return one_helper(qs, DOESNOTEXIST_MSG)
     else:
         assert idlike is None  # noqa: S101
@@ -210,24 +206,23 @@ def get(
         if issubclass(registry, IsVersioned) and is_latest_was_not_in_expressions:
             expressions["is_latest"] = True
         try:
-            return registry.objects.using(qs.db).get(**expressions)
-        except registry.DoesNotExist:
+            return BasicQuerySet.get(qs, **expressions)
+        except registry.DoesNotExist as e:
             # handle the case in which the is_latest injection led to a missed query
             if "is_latest" in expressions and is_latest_was_not_in_expressions:
                 expressions.pop("is_latest")
                 result = (
-                    registry.objects.using(qs.db)
-                    .filter(**expressions)
+                    BasicQuerySet.filter(qs, **expressions)
                     .order_by("-created_at")
                     .first()
                 )
                 if result is not None:
                     return result
-            raise registry.DoesNotExist from registry.DoesNotExist
+            raise registry.DoesNotExist from e
 
 
 class SQLRecordList(UserList, Generic[T]):
-    """Is ordered, can't be queried, but has `.df()`."""
+    """Is ordered, can't be queried, but has `.to_dataframe()`."""
 
     def __init__(self, records: Iterable[T]):
         if isinstance(records, list):
@@ -235,16 +230,24 @@ class SQLRecordList(UserList, Generic[T]):
         else:
             super().__init__(records)  # Let UserList handle the conversion
 
-    def df(self) -> pd.DataFrame:
+    def to_dataframe(self) -> pd.DataFrame:
         keys = get_keys_from_df(self.data, self.data[0].__class__)
         values = [record.__dict__ for record in self.data]
         return pd.DataFrame(values, columns=keys)
 
-    def list(
+    @deprecated(new_name="to_dataframe")
+    def df(self) -> pd.DataFrame:
+        return self.to_dataframe()
+
+    def to_list(
         self, field: str
-    ) -> list[str]:  # meaningful to be parallel with list() in QuerySet
+    ) -> list[str]:  # meaningful to be parallel with to_list() in QuerySet
         return [getattr(record, field) for record in self.data]
 
+    @deprecated(new_name="to_list")
+    def list(self, field: str) -> list[str]:
+        return self.to_list(field)
+
     def one(self) -> T:
         """Exactly one result. Throws error if there are more or none."""
         return one_helper(self)
@@ -348,7 +351,7 @@ def get_feature_annotate_kwargs(
             | Q(dtype__startswith="cat[ULabel")
             | Q(dtype__startswith="cat[Record")
         )
-        feature_names = feature_qs.list("name")
+        feature_names = feature_qs.to_list("name")
         logger.important(
             f"queried for all categorical features with dtype ULabel or Record and non-categorical features: ({len(feature_names)}) {feature_names}"
         )
@@ -671,8 +674,8 @@ class BasicQuerySet(models.QuerySet):
             new_cls = cls
         return object.__new__(new_cls)
 
-    @doc_args(SQLRecord.df.__doc__)
-    def df(
+    @doc_args(SQLRecord.to_dataframe.__doc__)
+    def to_dataframe(
         self,
         include: str | list[str] | None = None,
         features: bool | list[str] | str | None = None,
@@ -706,7 +709,7 @@ class BasicQuerySet(models.QuerySet):
             id_subquery = self.values("id")
             time = logger.debug("finished get id values", time=time)
             # for annotate, we want the queryset without filters so that joins don't affect the annotations
-            query_set_without_filters = self.model.objects.filter(
+            query_set_without_filters = self.model.objects.using(self._db).filter(
                 id__in=Subquery(id_subquery)
             )
             time = logger.debug("finished get query_set_without_filters", time=time)
@@ -739,26 +742,34 @@ class BasicQuerySet(models.QuerySet):
         time = logger.debug("finished", time=time)
         return df_reshaped
 
+    @deprecated(new_name="to_dataframe")
+    def df(
+        self,
+        include: str | list[str] | None = None,
+        features: bool | list[str] | str | None = None,
+    ) -> pd.DataFrame:
+        return self.to_dataframe(include, features)
+
     def delete(self, *args, **kwargs):
         """Delete all records in the query set."""
-        from lamindb.models import Artifact, Collection, Run, Transform
+        from lamindb.models import Artifact, Collection, Run, Storage, Transform
 
         # both Transform & Run might reference artifacts
-        if self.model in {Artifact, Collection, Transform, Run}:
+        if self.model in {Artifact, Collection, Transform, Run, Storage}:
             for record in self:
                 logger.important(f"deleting {record}")
                 record.delete(*args, **kwargs)
         else:
             super().delete(*args, **kwargs)
 
-    def list(self, field: str | None = None) -> list[SQLRecord] | list[str]:
+    def to_list(self, field: str | None = None) -> list[SQLRecord] | list[str]:
         """Populate an (unordered) list with the results.
 
         Note that the order in this list is only meaningful if you ordered the underlying query set with `.order_by()`.
 
         Examples:
-            >>> queryset.list()  # list of records
-            >>> queryset.list("name")  # list of values
+            >>> queryset.to_list()  # list of records
+            >>> queryset.to_list("name")  # list of values
         """
         if field is None:
             return list(self)
@@ -766,6 +777,10 @@ class BasicQuerySet(models.QuerySet):
             # list casting is necessary because values_list does not return a list
             return list(self.values_list(field, flat=True))
 
+    @deprecated(new_name="to_list")
+    def list(self, field: str | None = None) -> list[SQLRecord] | list[str]:
+        return self.to_list(field)
+
     def first(self) -> SQLRecord | None:
         """If non-empty, the first result in the query set, otherwise ``None``.
 
@@ -869,8 +884,18 @@ class QuerySet(BasicQuerySet):
         """Query a single record. Raises error if there are more or none."""
         is_run_input = expressions.pop("is_run_input", False)
 
+        if path := expressions.pop("path", None):
+            from .artifact_set import ArtifactSet, artifacts_from_path
+
+            if not isinstance(self, ArtifactSet):
+                raise ValueError("Querying by path is only possible for artifacts.")
+
+            qs = artifacts_from_path(self, path)
+        else:
+            qs = self
+
         try:
-            record = get(self, idlike, **expressions)
+            record = get(qs, idlike, **expressions)  # type: ignore
         except ValueError as e:
             # Pass through original error for explicit id lookups
             if "Field 'id' expected a number" in str(e):
@@ -886,8 +911,8 @@ class QuerySet(BasicQuerySet):
             raise  # pragma: no cover
 
         if is_run_input is not False:  # might be None or True or Run
-            from lamindb.models.artifact import Artifact, _track_run_input
-            from lamindb.models.collection import Collection
+            from .artifact import Artifact, _track_run_input
+            from .collection import Collection
 
             if isinstance(record, (Artifact, Collection)):
                 _track_run_input(record, is_run_input)

lamindb/models/record.py

@@ -20,7 +20,7 @@ from .can_curate import CanCurate
 from .feature import Feature
 from .has_parents import _query_relatives
 from .query_set import reorder_subset_columns_in_df
-from .run import Run, TracksRun, TracksUpdates
+from .run import Run, TracksRun, TracksUpdates, User
 from .sqlrecord import BaseSQLRecord, IsLink, SQLRecord, _get_record_kwargs
 from .transform import Transform
 from .ulabel import ULabel
@@ -54,6 +54,7 @@ class Record(SQLRecord, CanCurate, TracksRun, TracksUpdates):
 
     class Meta(SQLRecord.Meta, TracksRun.Meta, TracksUpdates.Meta):
         abstract = False
+        app_label = "lamindb"
 
     _name_field: str = "name"
 
@@ -207,11 +208,13 @@ class Record(SQLRecord, CanCurate, TracksRun, TracksUpdates):
     def to_pandas(self) -> pd.DataFrame:
         """Export all children of a record type recursively to a pandas DataFrame."""
         assert self.is_type, "Only types can be exported as dataframes"  # noqa: S101
-        df = self.query_children().df(features="queryset")
+        df = self.query_children().to_dataframe(features="queryset")
         df.columns.values[0] = "__lamindb_record_uid__"
         df.columns.values[1] = "__lamindb_record_name__"
         if self.schema is not None:
-            desired_order = self.schema.members.list("name")  # only members is ordered!
+            desired_order = self.schema.members.to_list(
+                "name"
+            )  # only members is ordered!
         else:
             # sort alphabetically for now
             desired_order = df.columns[2:].tolist()
@@ -235,7 +238,7 @@ class Record(SQLRecord, CanCurate, TracksRun, TracksUpdates):
         )
         run = Run(transform, initiated_by_run=context.run).save()
         run.input_records.add(self)
-        return Artifact.from_df(
+        return Artifact.from_dataframe(
             self.to_pandas(),
             key=key,
             description=f"Export of sheet {self.uid}{description}",
@@ -252,6 +255,7 @@ class RecordJson(BaseSQLRecord, IsLink):
     value: Any = JSONField(default=None, db_default=None)
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("record", "feature")  # a list is modeled as a list in json
 
 
@@ -266,6 +270,7 @@ class RecordRecord(SQLRecord, IsLink):
     )  # component
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("record", "feature", "value")
 
 
@@ -277,6 +282,19 @@ class RecordULabel(BaseSQLRecord, IsLink):
 
     class Meta:
         # allows linking exactly one record to one ulabel per feature, because we likely don't want to have Many
+        app_label = "lamindb"
+        unique_together = ("record", "feature", "value")
+
+
+class RecordUser(BaseSQLRecord, IsLink):
+    id: int = models.BigAutoField(primary_key=True)
+    record: Record = ForeignKey(Record, CASCADE, related_name="values_user")
+    feature: Feature = ForeignKey(Feature, PROTECT, related_name="links_recorduser")
+    value: User = ForeignKey(User, PROTECT, related_name="links_record")
+
+    class Meta:
+        # allows linking exactly one record to one user per feature, because we likely don't want to have Many
+        app_label = "lamindb"
         unique_together = ("record", "feature", "value")
 
 
@@ -288,6 +306,7 @@ class RecordRun(BaseSQLRecord, IsLink):
 
     class Meta:
         # allows linking several records to a single run for the same feature because we'll likely need this
+        app_label = "lamindb"
         unique_together = ("record", "feature", "value")
 
 
@@ -299,6 +318,7 @@ class RecordArtifact(BaseSQLRecord, IsLink):
 
     class Meta:
         # allows linking several records to a single artifact for the same feature because we'll likely need this
+        app_label = "lamindb"
         unique_together = ("record", "feature", "value")
 
 
@@ -315,4 +335,5 @@ class ArtifactRecord(BaseSQLRecord, IsLink):
 
     class Meta:
         # allows linking several records to a single artifact for the same feature because we'll likely need this
+        app_label = "lamindb"
         unique_together = ("artifact", "record", "feature")

lamindb/models/run.py

@@ -142,6 +142,9 @@ class User(BaseSQLRecord, CanCurate):
         >>> user
     """
 
+    class Meta:
+        app_label = "lamindb"
+
     _name_field: str = "handle"
 
     id: int = models.AutoField(primary_key=True)
@@ -223,6 +226,9 @@ class Run(SQLRecord):
         >>> ln.context.run
     """
 
+    class Meta:
+        app_label = "lamindb"
+
     _name_field: str = "started_at"
 
     id: int = models.BigAutoField(primary_key=True)
@@ -368,11 +374,6 @@ class Run(SQLRecord):
             reference_type=reference_type,
         )
 
-    def delete(self) -> None:
-        """Delete."""
-        delete_run_artifacts(self)
-        super().delete()
-
     @property
     @deprecated("features")
     def params(self) -> FeatureManager:
@@ -470,7 +471,7 @@ def delete_run_artifacts(run: Run) -> None:
         if environment._environment_of.count() == 0:
             environment.delete(permanent=True)
     if report is not None:
-        # only delete if there are no other runs attached to this environment
+        # only delete if there are no other runs attached to this report
         if report._report_of.count() == 0:
             report.delete(permanent=True)
 
@@ -492,4 +493,5 @@ class RunFeatureValue(BaseSQLRecord, IsLink):
     """Creator of record."""
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("run", "featurevalue")

lamindb/models/schema.py

@@ -6,6 +6,7 @@ import numpy as np
 from django.db import models
 from django.db.models import CASCADE, PROTECT, ManyToManyField
 from lamin_utils import logger
+from lamindb_setup.core import deprecated
 from lamindb_setup.core.hashing import HASH_LENGTH, hash_string
 from rich.table import Table
 from rich.text import Text
@@ -348,11 +349,12 @@ class Schema(SQLRecord, CanCurate, TracksRun):
 
             # from a dataframe
             df = pd.DataFrame({"feat1": [1, 2], "feat2": [3.1, 4.2], "feat3": ["cond1", "cond2"]})
-            schema = ln.Schema.from_df(df)
+            schema = ln.Schema.from_dataframe(df)
     """
 
     class Meta(SQLRecord.Meta, TracksRun.Meta, TracksUpdates.Meta):
         abstract = False
+        app_label = "lamindb"
 
     _name_field: str = "name"
     _aux_fields: dict[str, tuple[str, type]] = {
@@ -576,19 +578,22 @@ class Schema(SQLRecord, CanCurate, TracksRun):
                 self.optionals.set(optional_features)
                 return None
         self._slots: dict[str, Schema] = {}
+
         if features:
             self._features = (get_related_name(features_registry), features)  # type: ignore
-        elif slots:
+        if slots:
             for slot_key, component in slots.items():
                 if component._state.adding:
                     raise InvalidArgument(
                         f"schema for {slot_key} {component} must be saved before use"
                     )
             self._slots = slots
+
         if validated_kwargs["hash"] in KNOWN_SCHEMAS:
             validated_kwargs["uid"] = KNOWN_SCHEMAS[validated_kwargs["hash"]]
         else:
             validated_kwargs["uid"] = ids.base62_16()
+
         super().__init__(**validated_kwargs)
 
     def _validate_kwargs_calculate_hash(
@@ -623,14 +628,20 @@ class Schema(SQLRecord, CanCurate, TracksRun):
                 raise TypeError("index must be a Feature")
             features.insert(0, index)
 
+        if slots:
+            itype = "Composite"
+            if otype is None:
+                raise InvalidArgument("Please pass otype != None for composite schemas")
+
         if features:
             features, configs = get_features_config(features)
             features_registry = validate_features(features)
-            itype_compare = features_registry.__get_name_with_module__()
-            if itype is not None:
-                assert itype.startswith(itype_compare), str(itype_compare)  # noqa: S101
-            else:
-                itype = itype_compare
+            if itype != "Composite":
+                itype_compare = features_registry.__get_name_with_module__()
+                if itype is not None:
+                    assert itype.startswith(itype_compare), str(itype_compare)  # noqa: S101
+                else:
+                    itype = itype_compare
             if n_features is not None:
                 if n_features != len(features):
                     logger.important(f"updating to n {len(features)} features")
@@ -654,11 +665,6 @@ class Schema(SQLRecord, CanCurate, TracksRun):
         if flexible is None:
             flexible = flexible_default
 
-        if slots:
-            itype = "Composite"
-            if otype is None:
-                raise InvalidArgument("Please pass otype != None for composite schemas")
-
         if itype is not None and not isinstance(itype, str):
             itype_str = serialize_dtype(itype, is_itype=True)
         else:
@@ -771,7 +777,7 @@ class Schema(SQLRecord, CanCurate, TracksRun):
         cls,
         values: ListLike,
         field: FieldAttr = Feature.name,
-        type: str | None = None,
+        dtype: str | None = None,
         name: str | None = None,
         mute: bool = False,
         organism: SQLRecord | str | None = None,
@@ -783,7 +789,7 @@ class Schema(SQLRecord, CanCurate, TracksRun):
         Args:
             values: A list of values, like feature names or ids.
             field: The field of a reference registry to map values.
-            type: The simple type.
+            dtype: The simple dtype.
                 Defaults to `None` if reference registry is :class:`~lamindb.Feature`,
                 defaults to `"float"` otherwise.
             name: A name.
@@ -816,8 +822,8 @@ class Schema(SQLRecord, CanCurate, TracksRun):
         if isinstance(values, DICT_KEYS_TYPE):
             values = list(values)
         registry = field.field.model
-        if registry != Feature and type is None:
-            type = NUMBER_TYPE
+        if registry != Feature and dtype is None:
+            dtype = NUMBER_TYPE
             logger.debug("setting feature set to 'number'")
         validated = registry.validate(values, field=field, mute=mute, organism=organism)
         values_array = np.array(values)
@@ -841,12 +847,12 @@ class Schema(SQLRecord, CanCurate, TracksRun):
         schema = Schema(
             features=validated_features,
             name=name,
-            dtype=get_type_str(type),
+            dtype=get_type_str(dtype),
         )
         return schema
 
     @classmethod
-    def from_df(
+    def from_dataframe(
         cls,
         df: pd.DataFrame,
         field: FieldAttr = Feature.name,
@@ -889,15 +895,28 @@ class Schema(SQLRecord, CanCurate, TracksRun):
             )
         return schema
 
+    @classmethod
+    @deprecated("from_dataframe")
+    def from_df(
+        cls,
+        df: pd.DataFrame,
+        field: FieldAttr = Feature.name,
+        name: str | None = None,
+        mute: bool = False,
+        organism: SQLRecord | str | None = None,
+        source: SQLRecord | None = None,
+    ) -> Schema | None:
+        return cls.from_dataframe(df, field, name, mute, organism, source)
+
     def save(self, *args, **kwargs) -> Schema:
-        """Save."""
+        """Save schema."""
         from .save import bulk_create
 
         if self.pk is not None:
             features = (
                 self._features[1]
                 if hasattr(self, "_features")
-                else (self.members.list() if self.members.exists() else [])
+                else (self.members.to_list() if self.members.exists() else [])
             )
             index_feature = self.index
             _, validated_kwargs, _, _, _ = self._validate_kwargs_calculate_hash(
@@ -925,7 +944,7 @@ class Schema(SQLRecord, CanCurate, TracksRun):
                 datasets = Artifact.filter(schema=self).all()
                 if datasets.exists():
                     logger.warning(
-                        f"you updated the schema hash and might invalidate datasets that were previously validated with this schema: {datasets.list('uid')}"
+                        f"you updated the schema hash and might invalidate datasets that were previously validated with this schema: {datasets.to_list('uid')}"
                     )
                 self.hash = validated_kwargs["hash"]
                 self.n = validated_kwargs["n"]
@@ -947,13 +966,16 @@ class Schema(SQLRecord, CanCurate, TracksRun):
             assert self.n > 0  # noqa: S101
             using: bool | None = kwargs.pop("using", None)
             related_name, records = self._features
+
+            # .set() does not preserve the order but orders by the feature primary key
             # only the following method preserves the order
-            # .set() does not preserve the order but orders by
-            # the feature primary key
             through_model = getattr(self, related_name).through
-            related_model_split = parse_cat_dtype(self.itype, is_itype=True)[
-                "registry_str"
-            ].split(".")
+            if self.itype == "Composite":
+                related_model_split = ["Feature"]
+            else:
+                related_model_split = parse_cat_dtype(self.itype, is_itype=True)[
+                    "registry_str"
+                ].split(".")
             if len(related_model_split) == 1:
                 related_field = related_model_split[0].lower()
             else:
@@ -965,6 +987,7 @@ class Schema(SQLRecord, CanCurate, TracksRun):
             ]
             through_model.objects.using(using).bulk_create(links, ignore_conflicts=True)
             delattr(self, "_features")
+
         return self
 
     @property
@@ -978,6 +1001,8 @@ class Schema(SQLRecord, CanCurate, TracksRun):
             # this should return a queryset and not a list...
             # need to fix this
             return self._features[1]
+        if len(self.features.all()) > 0:
+            return self.features.order_by("links_schema__id")
         if self.itype == "Composite" or self.is_type:
             return Feature.objects.none()
         related_name = self._get_related_name()
@@ -1200,6 +1225,7 @@ class SchemaFeature(BaseSQLRecord, IsLink):
     feature: Feature = ForeignKey(Feature, PROTECT, related_name="links_schema")
 
     class Meta:
+        app_label = "lamindb"
         unique_together = ("schema", "feature")
 
 
@@ -1211,6 +1237,7 @@ class ArtifactSchema(BaseSQLRecord, IsLink, TracksRun):
     feature_ref_is_semantic: bool | None = BooleanField(null=True)
 
     class Meta:
+        app_label = "lamindb"
         unique_together = (("artifact", "schema"), ("artifact", "slot"))
 
 
@@ -1221,6 +1248,7 @@ class SchemaComponent(BaseSQLRecord, IsLink, TracksRun):
     slot: str | None = CharField(null=True)
 
     class Meta:
+        app_label = "lamindb"
         unique_together = (("composite", "slot", "component"), ("composite", "slot"))